#include "SZLOrderedPartitionedZoneWriterMPI.h"
#include "ThirdPartyHeadersBegin.h"
#include <limits>
#include <boost/foreach.hpp>
#include <boost/functional/hash.hpp>
#include <boost/make_shared.hpp>
#include <boost/unordered_map.hpp>
#include <boost/unordered_set.hpp>
#include "ThirdPartyHeadersEnd.h"
#include "FieldData.h"
#include "FileWriterInterface.h"
#include "ItemSetIterator.h"
#include "MPICommunicationCache.h"
#include "MPICommunicator.h"
#include "MPIFileWriter.h"
#include "MPINonBlockingCommunicationCollection.h"
#include "MPIUtil.h"
#include "SZLOrderedPartitionWriter.h"
#include "TecioMPI.h"
#include "ZoneInfoCache.h"
using namespace tecplot::___3933; namespace tecplot { namespace ___3933 { std::size_t hash_value(___1844 const& ___1843) { std::size_t seed = 0; boost::hash_combine(seed, ___1843.i()); boost::hash_combine(seed, ___1843.___2105()); boost::hash_combine(seed, ___1843.___2134()); return seed; } }} namespace tecplot { namespace teciompi { struct SZLOrderedPartitionedZoneWriterMPI::Impl { typedef boost::unordered_map<___2090::SubzoneOffset_t, std::vector<___2479> > SubzoneMinMaxMap; struct NeighborExtents { SimpleVector<___2090::___2980> neighborPartitions; SimpleVector<uint64_t> neighborMinNodeNumbers; SimpleVector<uint64_t> neighborMaxNodeNumbers; }; typedef boost::unordered_map<___2090::___2980, NeighborExtents> NeighborExtentsMap; struct IJKExtents { ___1844 ___2474; ___1844 ___2364; IJKExtents() {} IJKExtents(___1844 const& ___2474, ___1844 const& ___2364) : ___2474(___2474) , ___2364(___2364) {} }; boost::unordered_map<___2090::___2980, IJKExtents> partitionExtentsMap; struct PartitionIntersection { ___2090::___2980 ___2977; ___2090::___2980 neighborPartition; IJKExtents extents; PartitionIntersection( ___2090::___2980 ___2977, ___2090::___2980 neighborPartition, ___1844 const& ___2474, ___1844 const& ___2364) : ___2977(___2977) , neighborPartition(neighborPartition) , extents(___2474, ___2364) {} }; struct SendOrder { bool operator() (PartitionIntersection const& a, PartitionIntersection const& b) { if (a.___2977 < b.___2977) return true; else if (a.___2977 == b.___2977 && a.neighborPartition < b.neighborPartition) return true; return false; } }; struct ReceiveOrder { bool operator() (PartitionIntersection const& a, PartitionIntersection const& b) { if (a.neighborPartition < b.neighborPartition) return true; else if (a.neighborPartition == b.neighborPartition && a.___2977 < b.___2977) return true; return false; } }; struct SubzoneMinMaxes { SimpleVector<___2090::SubzoneOffset_t> subzones; SimpleVector<___2479> varMinMaxes; }; typedef boost::unordered_map<___2090::___2980, SubzoneMinMaxes> NeighborMinMaxMap; typedef boost::unordered_map<___2090::___2980, NeighborMinMaxMap> PartitionMinMaxMap; SZLOrderedPartitionedZoneWriterMPI& m_outer; MPI_Comm m_comm; int m_mainProcess; int m_localProcess; bool m_useCollectiveRoutines; Impl( SZLOrderedPartitionedZoneWriterMPI& outer, MPI_Comm comm, int mainProcess, ___37& ___36, ___4636 zone) : m_outer(outer) , m_comm(comm) , m_mainProcess(mainProcess) , m_useCollectiveRoutines(everyRankOwnsOnePartition(comm, ___36, zone)) { MPI_Comm_rank(m_comm, &m_localProcess); createPartitionWriters(); } typedef std::map<___2090::___2980, boost::shared_ptr<___3933::___1881> > PartitionIJKInfoMap; void createPartitionWriters(); void sendLocalPartitionExtentsToMainProcess(PartitionIJKInfoMap& partitionIJKInfoMap, MPINonBlockingCommunicationCollection& communicationCollection); void collectAllPartitionExtents(MPINonBlockingCommunicationCollection& communicationCollection); void distributePartitionNeighborExtents(MPINonBlockingCommunicationCollection& communicationCollection); void receiveNeighborPartitionExtents( PartitionIJKInfoMap const& partitionijkInfoMap, MPINonBlockingCommunicationCollection& communicationCollection, NeighborExtentsMap& neighborExtentsMap); void calculateNeighborIntesections( NeighborExtentsMap const& neighborExtentsMap, std::vector<PartitionIntersection>& partitionIntersections); void applyCellMinMaxToNodeSubzones( ___2227 cellI, ___2227 cellJ, ___2227 cellK, boost::unordered_set<___1844> const& nodeSet, IJKSubzoneInfo const& subzoneInfo, ___1844 const& partitionMinIJK, std::vector<___2479> const& cellMinMaxes, SubzoneMinMaxMap& nszMinMaxMap); void sendMinMaxesToNeighbors( std::vector<PartitionIntersection>& partitionIntersections, MPINonBlockingCommunicationCollection& communicationCollection); void receiveMinMaxesFromNeighbors( std::vector<PartitionIntersection> const& partitionIntersections, MPINonBlockingCommunicationCollection& communicationCollection, PartitionMinMaxMap& partitionMinMaxMap); void applyReceivedMinMaxesToSubzones( PartitionMinMaxMap const& partitionMinMaxMap, size_t ___2843, PartitionIJKInfoMap& partitionIJKInfoMap); void exchangeGhostInfo(PartitionIJKInfoMap& partitionIJKInfoMap); }; SZLOrderedPartitionedZoneWriterMPI::SZLOrderedPartitionedZoneWriterMPI( ItemSetIterator& varIter, ___4636 zone, ___4636 ___341, std::vector<___372> const& ___4564, ___372 ___4499, ___37& ___36, ZoneInfoCache& zoneInfoCache, MPI_Comm communicator, int mainProcess) : SZLOrderedPartitionedZoneWriter(varIter, zone, ___341, ___4564, ___4499, ___36, zoneInfoCache) , m_impl(new Impl(*this, communicator, mainProcess, ___36, zone)) {} SZLOrderedPartitionedZoneWriterMPI::~SZLOrderedPartitionedZoneWriterMPI() {} ___2479 SZLOrderedPartitionedZoneWriterMPI::varMinMax(___3933::___4352 datasetVar) { REQUIRE(m_varIter.baseItem() <= datasetVar && datasetVar < m_varIter.___2812());
___2479 ___3358; ___4352 const baseVar = m_varIter.baseItem(); for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) ___3358.include(m_varPartitionMinMaxes[datasetVar - baseVar][___2977]); return ___3358; } namespace { std::vector<___1864> gatherPartitionTreeItems( UInt64Array const& minNodeNumbers, UInt64Array const& maxNodeNumbers, ___1844 const& zoneSize, ___2090::___2980 numPartitions) { std::vector<___1864> ___2981; ___2981.reserve(numPartitions); for (___2090::___2980 ___2977 = 0; ___2977 < numPartitions; ++___2977) { ___1844 partitionMinIJK = zoneSize.ijkAtOffset(static_cast<___81>(minNodeNumbers[___2977])); ___1844 partitionMaxIJK = (zoneSize.ijkAtOffset(static_cast<___81>(maxNodeNumbers[___2977])) + 1).minOp(zoneSize - 1); ___1853 ___2478(partitionMinIJK.i(), partitionMinIJK.___2105(), partitionMinIJK.___2134()); ___1853 ___2372(partitionMaxIJK.i(), partitionMaxIJK.___2105(), partitionMaxIJK.___2134()); ___2981.push_back(std::make_pair(___1855(___2478, ___2372), ___2977)); } return ___2981; } } void SZLOrderedPartitionedZoneWriterMPI::Impl::sendLocalPartitionExtentsToMainProcess( PartitionIJKInfoMap& partitionIJKInfoMap, MPINonBlockingCommunicationCollection& communicationCollection) { ___1844 zoneSize; m_outer.___2337.___4615(m_outer.___2677 + 1, zoneSize); BOOST_FOREACH(PartitionIJKInfoMap::value_type const& infoPair, partitionIJKInfoMap) { communicationCollection.sendScalar(m_outer.m_partitionMinNodeNumbers[infoPair.first], m_mainProcess, TecioMPI::PARTITION_MIN_NODE_NUMBER); communicationCollection.sendScalar(m_outer.m_partitionMaxNodeNumbers[infoPair.first], m_mainProcess, TecioMPI::PARTITION_MAX_NODE_NUMBER); } } void SZLOrderedPartitionedZoneWriterMPI::Impl::collectAllPartitionExtents(MPINonBlockingCommunicationCollection& communicationCollection) { for (___2090::___2980 ___2977 = 0; ___2977 < static_cast<___2090::___2980>(m_outer.___2337.zoneGetNumPartitions(m_outer.___2677 + 1)); ++___2977) { int32_t owner = m_outer.___2337.zonePartitionGetOwnerProcess(m_outer.___2677 + 1, ___2977 + 1); if (owner != m_localProcess) { communicationCollection.receiveScalar(m_outer.m_partitionMinNodeNumbers[___2977], owner, TecioMPI::PARTITION_MIN_NODE_NUMBER); communicationCollection.receiveScalar(m_outer.m_partitionMaxNodeNumbers[___2977], owner, TecioMPI::PARTITION_MAX_NODE_NUMBER); } } communicationCollection.___4446(); } void SZLOrderedPartitionedZoneWriterMPI::Impl::distributePartitionNeighborExtents(MPINonBlockingCommunicationCollection& communicationCollection) { ___1844 zoneSize; m_outer.___2337.___4615(m_outer.___2677 + 1, zoneSize); ___2090::___2980 numPartitions = m_outer.___2337.zoneGetNumPartitions(m_outer.___2677 + 1); ___1863 partitionTree(gatherPartitionTreeItems(m_outer.m_partitionMinNodeNumbers, m_outer.m_partitionMaxNodeNumbers, zoneSize, numPartitions)); std::vector<___2090::___2980> neighborPartitions; std::vector<uint64_t> neighborMinNodes; std::vector<uint64_t> neighborMaxNodes; std::vector<___1864> neighborItems; for (___2090::___2980 ___2977 = 0; ___2977 < numPartitions; ++___2977) { ___1844 partitionMinIJK = zoneSize.ijkAtOffset(static_cast<___81>(m_outer.m_partitionMinNodeNumbers[___2977])); ___1844 partitionMaxIJK = (zoneSize.ijkAtOffset(static_cast<___81>(m_outer.m_partitionMaxNodeNumbers[___2977])) + 1).minOp(zoneSize - 1); neighborItems.clear(); partitionTree.find(partitionMinIJK, partitionMaxIJK, neighborItems); neighborPartitions.clear(); neighborMinNodes.clear(); neighborMaxNodes.clear(); neighborPartitions.reserve(neighborItems.size()); neighborMinNodes.reserve(neighborItems.size()); neighborMaxNodes.reserve(neighborItems.size()); int32_t owner = m_outer.___2337.zonePartitionGetOwnerProcess(m_outer.___2677 + 1, ___2977 + 1); BOOST_FOREACH(___1864 const& neighborItem, neighborItems) { ___2090::___2980 neighborPartition = neighborItem.second; if (neighborPartition != ___2977) { neighborPartitions.push_back(neighborPartition); neighborMinNodes.push_back(m_outer.m_partitionMinNodeNumbers[neighborPartition]); neighborMaxNodes.push_back(m_outer.m_partitionMaxNodeNumbers[neighborPartition]); } } communicationCollection.sendVectorCopy(neighborPartitions, owner, TecioMPI::PARTITION_NEIGHBORS_SIZE, TecioMPI::PARTITION_NEIGHBORS_VEC); communicationCollection.sendVectorCopy(neighborMinNodes, owner, TecioMPI::PARTITION_NEIGHBOR_MIN_NODES_SIZE, TecioMPI::PARTITION_NEIGHBOR_MIN_NODES_VEC); communicationCollection.sendVectorCopy(neighborMaxNodes, owner, TecioMPI::PARTITION_NEIGHBOR_MAX_NODES_SIZE, TecioMPI::PARTITION_NEIGHBOR_MAX_NODES_VEC); } } void SZLOrderedPartitionedZoneWriterMPI::Impl::receiveNeighborPartitionExtents( PartitionIJKInfoMap const& partitionIJKInfoMap, MPINonBlockingCommunicationCollection& communicationCollection,
NeighborExtentsMap& neighborExtentsMap) { BOOST_FOREACH(PartitionIJKInfoMap::value_type const& infoPair, partitionIJKInfoMap) { ___2090::___2980 ___2977 = infoPair.first; communicationCollection.receiveVector(neighborExtentsMap[___2977].neighborPartitions, m_mainProcess, TecioMPI::PARTITION_NEIGHBORS_SIZE, TecioMPI::PARTITION_NEIGHBORS_VEC); communicationCollection.receiveVector(neighborExtentsMap[___2977].neighborMinNodeNumbers, m_mainProcess, TecioMPI::PARTITION_NEIGHBOR_MIN_NODES_SIZE, TecioMPI::PARTITION_NEIGHBOR_MIN_NODES_VEC); communicationCollection.receiveVector(neighborExtentsMap[___2977].neighborMaxNodeNumbers, m_mainProcess, TecioMPI::PARTITION_NEIGHBOR_MAX_NODES_SIZE, TecioMPI::PARTITION_NEIGHBOR_MAX_NODES_VEC); } communicationCollection.___4446(); } void SZLOrderedPartitionedZoneWriterMPI::Impl::calculateNeighborIntesections( NeighborExtentsMap const& neighborExtentsMap, std::vector<PartitionIntersection>& partitionIntersections) { REQUIRE(partitionIntersections.empty()); ___1844 zoneSize; m_outer.___2337.___4615(m_outer.___2677 + 1, zoneSize); BOOST_FOREACH(NeighborExtentsMap::value_type const& neighborPair, neighborExtentsMap) { ___2090::___2980 ___2977 = neighborPair.first; ___1844 partitionMinIJK; ___1844 partitionMaxIJK; m_outer.getPartitionExtentsWithGhostNodes(___2977, partitionMinIJK, partitionMaxIJK); partitionExtentsMap[___2977] = IJKExtents(partitionMinIJK, partitionMaxIJK); NeighborExtents const& neighborExtents = neighborPair.second; for (int i = 0; i < neighborExtents.neighborPartitions.size(); ++i) { ___2090::___2980 neighborPartition = neighborExtents.neighborPartitions[i]; ___1844 neighborMinIJK = zoneSize.ijkAtOffset(static_cast<___81>(neighborExtents.neighborMinNodeNumbers[i])); ___1844 neighborMaxIJK = (zoneSize.ijkAtOffset(static_cast<___81>(neighborExtents.neighborMaxNodeNumbers[i])) + 1).minOp(zoneSize - 1); partitionExtentsMap[neighborPartition] = IJKExtents(neighborMinIJK, neighborMaxIJK); ___1853 partitionMinCorner(partitionMinIJK.i(), partitionMinIJK.___2105(), partitionMinIJK.___2134()); ___1853 partitionMaxCorner(partitionMaxIJK.i(), partitionMaxIJK.___2105(), partitionMaxIJK.___2134()); ___1855 partitionRange(partitionMinCorner, partitionMaxCorner); ___1853 neighborMinCorner(neighborMinIJK.i(), neighborMinIJK.___2105(), neighborMinIJK.___2134()); ___1853 neighborMaxCorner(neighborMaxIJK.i(), neighborMaxIJK.___2105(), neighborMaxIJK.___2134()); ___1855 neighborRange(neighborMinCorner, neighborMaxCorner); ___1855 intersectionRange; boost::geometry::intersection(partitionRange, neighborRange, intersectionRange); m_outer.throwIfBadIntersectionRange(intersectionRange, ___2977, neighborPartition); partitionIntersections.push_back( PartitionIntersection( ___2977, neighborPartition, ___1844(static_cast<___81>(intersectionRange.min_corner().get<0>()), static_cast<___81>(intersectionRange.min_corner().get<1>()), static_cast<___81>(intersectionRange.min_corner().get<2>())), ___1844(static_cast<___81>(intersectionRange.max_corner().get<0>()), static_cast<___81>(intersectionRange.max_corner().get<1>()), static_cast<___81>(intersectionRange.max_corner().get<2>())))); } } } void SZLOrderedPartitionedZoneWriterMPI::Impl::applyCellMinMaxToNodeSubzones( ___2227 cellI, ___2227 cellJ, ___2227 cellK, boost::unordered_set<___1844> const& nodeSet, IJKSubzoneInfo const& subzoneInfo, ___1844 const& partitionMinIJK, std::vector<___2479> const& cellMinMaxes, SubzoneMinMaxMap& nszMinMaxMap) { for (___2227 ___2134 = 0; ___2134 <= 1; ++___2134) { for (___2227 ___2105 = 0; ___2105 <= 1; ++___2105) { for (___2227 i = 0; i <= 1; ++i) { ___1844 nodeIJK(static_cast<___81>(cellI + i), static_cast<___81>(cellJ + ___2105), static_cast<___81>(cellK + ___2134)); if (nodeSet.find(nodeIJK) != nodeSet.end()) { ___1844 partitionNodeIJK = nodeIJK - partitionMinIJK; ___2090::SubzoneOffset_t ___3880 = subzoneInfo.szAddressAtItemIJK(partitionNodeIJK).subzoneOffset(); if (nszMinMaxMap[___3880].empty()) nszMinMaxMap[___3880].resize(cellMinMaxes.size()); for (size_t m = 0; m < cellMinMaxes.size(); ++m) { nszMinMaxMap[___3880][m].include(cellMinMaxes[m]); } } } } } } void SZLOrderedPartitionedZoneWriterMPI::Impl::sendMinMaxesToNeighbors( std::vector<PartitionIntersection>& partitionIntersections, MPINonBlockingCommunicationCollection& communicationCollection) { ___1844 zoneSize; m_outer.___2337.___4615(m_outer.___2677 + 1, zoneSize); PartitionTecUtilDecorator partitionTecUtilDecorator(m_outer.___2337, m_outer.___2677 + 1); ___4352 const numVarsToWrite = m_outer.m_varIter.___2812(); for (size_t i = 0; i < partitionIntersections.size(); ++i) { ___2090::___2980 ___2977 = partitionIntersections[i].___2977; ___2090::___2980 neighborPartition = partitionIntersections[i].neighborPartition; IJKExtents intersectionExtents = partitionIntersections[i].extents; IJKExtents const& partitionExtents = partitionExtentsMap[___2977]; IJKExtents const& neighborExtents = partitionExtentsMap[neighborPartition];
if (intersectionExtents.___2474.i() == partitionExtents.___2364.i() || intersectionExtents.___2474.___2105() == partitionExtents.___2364.___2105() || intersectionExtents.___2474.___2134() == partitionExtents.___2364.___2134()) { if (intersectionExtents.___2364.i() == neighborExtents.___2364.i() && intersectionExtents.___2364.i() < zoneSize.i() - 1) intersectionExtents.___2364.setI(intersectionExtents.___2364.i() - 1); if (intersectionExtents.___2364.___2105() == neighborExtents.___2364.___2105() && intersectionExtents.___2364.___2105() < zoneSize.___2105() - 1) intersectionExtents.___2364.setJ(intersectionExtents.___2364.___2105() - 1); if (intersectionExtents.___2364.___2134() == neighborExtents.___2364.___2134() && intersectionExtents.___2364.___2134() < zoneSize.___2134() - 1) intersectionExtents.___2364.___3497(intersectionExtents.___2364.___2134() - 1); if (intersectionExtents.___2474 <= intersectionExtents.___2364) { std::vector<___1352> nodalFieldDatas(numVarsToWrite); m_outer.retrieveNodalFieldDataPtrsForPartition(partitionTecUtilDecorator, ___2977, nodalFieldDatas); int32_t owner = m_outer.___2337.zonePartitionGetOwnerProcess(m_outer.___2677 + 1, neighborPartition + 1); boost::unordered_set<___1844> nodeSet; for (___2227 ___2134 = intersectionExtents.___2474.___2134(); ___2134 <= intersectionExtents.___2364.___2134(); ++___2134) { for (___2227 ___2105 = intersectionExtents.___2474.___2105(); ___2105 <= intersectionExtents.___2364.___2105(); ++___2105) { for (___2227 i = intersectionExtents.___2474.i(); i <= intersectionExtents.___2364.i(); ++i) { ___1844 nodeIJK(static_cast<___81>(i), static_cast<___81>(___2105), static_cast<___81>(___2134)); nodeSet.insert(nodeIJK); } } } SubzoneMinMaxMap neighborMinMaxMap; ___1844 cellMinIJK( std::max(partitionExtents.___2474.i(), intersectionExtents.___2474.i() == 0 ? 0 : intersectionExtents.___2474.i() - 1), std::max(partitionExtents.___2474.___2105(), intersectionExtents.___2474.___2105() == 0 ? 0 : intersectionExtents.___2474.___2105() - 1), std::max(partitionExtents.___2474.___2134(), intersectionExtents.___2474.___2134() == 0 ? 0 : intersectionExtents.___2474.___2134() - 1)); ___1844 cellMaxIJK( std::min(partitionExtents.___2364.i() - 1, intersectionExtents.___2364.i()), std::min(partitionExtents.___2364.___2105() - 1, intersectionExtents.___2364.___2105()), std::min(partitionExtents.___2364.___2134() - 1, intersectionExtents.___2364.___2134())); std::vector<___2479> cellMinMaxes(nodalFieldDatas.size()); ___1844 partitionDimensionsIJK = partitionExtents.___2364 - partitionExtents.___2474 + 1; ___1844 neighborDimensionsIJK = neighborExtents.___2364 - neighborExtents.___2474 + 1; if (neighborExtents.___2364.i() < zoneSize.i() - 1) neighborDimensionsIJK.setI(neighborDimensionsIJK.i() - 1); if (neighborExtents.___2364.___2105() < zoneSize.___2105() - 1) neighborDimensionsIJK.setJ(neighborDimensionsIJK.___2105() - 1); if (neighborExtents.___2364.___2134() < zoneSize.___2134() - 1) neighborDimensionsIJK.___3497(neighborDimensionsIJK.___2134() - 1); IJKSubzoneInfo neighborSubzoneInfo(neighborDimensionsIJK, DEFAULT_SUBZONE_MAX_IJK_DIM, neighborPartition, true); for (___2227 cellK = cellMinIJK.___2134(); cellK <= cellMaxIJK.___2134(); ++cellK) { for (___2227 cellJ = cellMinIJK.___2105(); cellJ <= cellMaxIJK.___2105(); ++cellJ) { for (___2227 cellI = cellMinIJK.i(); cellI <= cellMaxIJK.i(); ++cellI) { ___2227 ___462 = ((cellK - partitionExtents.___2474.___2134()) * partitionDimensionsIJK.___2105() + cellJ - partitionExtents.___2474.___2105()) * partitionDimensionsIJK.i() + cellI - partitionExtents.___2474.i(); m_outer.getCellMinMaxes(cellMinMaxes, ___462, partitionDimensionsIJK, nodalFieldDatas); applyCellMinMaxToNodeSubzones(cellI, cellJ, cellK, nodeSet, neighborSubzoneInfo, neighborExtents.___2474, cellMinMaxes, neighborMinMaxMap); } } } std::vector<___2090::SubzoneOffset_t> nodeSubzones; std::vector<___2479> minMaxValues; nodeSubzones.reserve(neighborMinMaxMap.size()); minMaxValues.reserve(neighborMinMaxMap.size() * nodalFieldDatas.size()); BOOST_FOREACH(SubzoneMinMaxMap::value_type const& valuePair, neighborMinMaxMap) { nodeSubzones.push_back(valuePair.first); for (size_t i = 0; i < nodalFieldDatas.size(); ++i) minMaxValues.push_back(valuePair.second[i]); } communicationCollection.sendVectorCopy(nodeSubzones, owner, TecioMPI::REFERENCED_NODE_SUBZONES_SIZE, TecioMPI::REFERENCED_NODE_SUBZONES_VEC); communicationCollection.sendVectorCopy(minMaxValues, owner, TecioMPI::NODE_SUBZONE_MINMAX_VALUES_SIZE, TecioMPI::NODE_SUBZONE_MINMAX_VALUES_VEC); } } } } void SZLOrderedPartitionedZoneWriterMPI::Impl::receiveMinMaxesFromNeighbors( std::vector<PartitionIntersection> const& partitionIntersections, MPINonBlockingCommunicationCollection& communicationCollection, PartitionMinMaxMap& partitionMinMaxMap) { ___1844 zoneSize; m_outer.___2337.___4615(m_outer.___2677 + 1, zoneSize); for (size_t i = 0; i < partitionIntersections.size(); ++i) { ___2090::___2980 const ___2977 = partitionIntersections[i].___2977; ___2090::___2980 const neighborPartition = partitionIntersections[i].neighborPartition; IJKExtents intersectionExtents = partitionIntersections[i].extents;
IJKExtents const& partitionExtents = partitionExtentsMap[___2977]; if (intersectionExtents.___2364.i() == partitionExtents.___2364.i() && intersectionExtents.___2364.i() < zoneSize.i() - 1) intersectionExtents.___2364.setI(intersectionExtents.___2364.i() - 1); if (intersectionExtents.___2364.___2105() == partitionExtents.___2364.___2105() && intersectionExtents.___2364.___2105() < zoneSize.___2105() - 1) intersectionExtents.___2364.setJ(intersectionExtents.___2364.___2105() - 1); if (intersectionExtents.___2364.___2134() == partitionExtents.___2364.___2134() && intersectionExtents.___2364.___2134() < zoneSize.___2134() - 1) intersectionExtents.___2364.___3497(intersectionExtents.___2364.___2134() - 1); if (intersectionExtents.___2474 <= intersectionExtents.___2364) { int32_t owner = m_outer.___2337.zonePartitionGetOwnerProcess(m_outer.___2677 + 1, neighborPartition + 1); communicationCollection.receiveVector(partitionMinMaxMap[___2977][neighborPartition].subzones, owner, TecioMPI::REFERENCED_NODE_SUBZONES_SIZE, TecioMPI::REFERENCED_NODE_SUBZONES_VEC); communicationCollection.receiveVector(partitionMinMaxMap[___2977][neighborPartition].varMinMaxes, owner, TecioMPI::NODE_SUBZONE_MINMAX_VALUES_SIZE, TecioMPI::NODE_SUBZONE_MINMAX_VALUES_VEC); } } } void SZLOrderedPartitionedZoneWriterMPI::Impl::applyReceivedMinMaxesToSubzones( PartitionMinMaxMap const& partitionMinMaxMap, size_t ___2843, PartitionIJKInfoMap& partitionIJKInfoMap) { BOOST_FOREACH(PartitionMinMaxMap::value_type const& partitionNeighborPair, partitionMinMaxMap) { ___2090::___2980 ___2977 = partitionNeighborPair.first; boost::shared_ptr<___1881> const& partitionIJKInfoPtr = partitionIJKInfoMap[___2977]; BOOST_FOREACH(NeighborMinMaxMap::value_type const& neighborMinMaxPair, partitionNeighborPair.second) { ___2090::___2980 neighborPartition = neighborMinMaxPair.first; SubzoneMinMaxes const& subzoneMinMaxes = neighborMinMaxPair.second; if (subzoneMinMaxes.varMinMaxes.size() != static_cast<int>(___2843) * subzoneMinMaxes.subzones.size()) { std::ostringstream ___2892; ___2892 << "Mismatched var minmax array sizes (internal error)." << " Partition: " << ___2977 << " Neighbor partition: " << neighborPartition << " # subzones: " << subzoneMinMaxes.subzones.size() << " # vars: " << ___2843 << " # varMinMaxes: " << subzoneMinMaxes.varMinMaxes.size() << ". Please contact technical support."; throw std::runtime_error(___2892.str()); } std::vector<___2479> minMaxesForNode(___2843); for (int i = 0; i < subzoneMinMaxes.subzones.size(); ++i) { if (!partitionIJKInfoPtr->validNodeSubzone(subzoneMinMaxes.subzones[i])) { std::ostringstream ___2892; ___2892 << "Invalid node subzone (" << subzoneMinMaxes.subzones[i] << ") passed from partition " << neighborPartition << " to partition " << ___2977 << " (internal error). Please contact technical support."; throw std::runtime_error(___2892.str()); } for (size_t ___2105 = 0; ___2105 < ___2843; ++___2105) { ___478(i * ___2843 + ___2105 <= static_cast<size_t>(std::numeric_limits<int32_t>::max())); int ind = static_cast<int>(i * ___2843 + ___2105); minMaxesForNode[___2105] = subzoneMinMaxes.varMinMaxes[ind]; } partitionIJKInfoPtr->includeNszVarMinMax(subzoneMinMaxes.subzones[i], minMaxesForNode); } } } } void SZLOrderedPartitionedZoneWriterMPI::Impl::exchangeGhostInfo(PartitionIJKInfoMap& partitionIJKInfoMap) { MPINonBlockingCommunicationCollection communicationCollection(m_comm); if (m_localProcess == m_mainProcess) { collectAllPartitionExtents(communicationCollection); distributePartitionNeighborExtents(communicationCollection); } else { sendLocalPartitionExtentsToMainProcess(partitionIJKInfoMap, communicationCollection); } NeighborExtentsMap neighborExtentsMap; receiveNeighborPartitionExtents(partitionIJKInfoMap, communicationCollection, neighborExtentsMap); std::vector<PartitionIntersection> partitionIntersections; calculateNeighborIntesections(neighborExtentsMap, partitionIntersections); std::sort(partitionIntersections.begin(), partitionIntersections.end(), SendOrder()); sendMinMaxesToNeighbors(partitionIntersections, communicationCollection); std::sort(partitionIntersections.begin(), partitionIntersections.end(), ReceiveOrder()); PartitionMinMaxMap partitionMinMaxMap; receiveMinMaxesFromNeighbors(partitionIntersections, communicationCollection, partitionMinMaxMap); communicationCollection.___4446(); applyReceivedMinMaxesToSubzones(partitionMinMaxMap, m_outer.m_varIter.___2812(), partitionIJKInfoMap); } void SZLOrderedPartitionedZoneWriterMPI::Impl::createPartitionWriters() { PartitionIJKInfoMap partitionIJKInfoMap; ___2090::___2980 numPartitions = static_cast<___2090::___2980>(m_outer.___2337.zoneGetNumPartitions(m_outer.___2677 + 1)); ___1844 zoneSize; m_outer.___2337.___4615(m_outer.___2677 + 1, zoneSize); for (___2090::___2980 ___2977 = 0; ___2977 < numPartitions; ++___2977) { if (m_outer.___2337.zonePartitionGetOwnerProcess(m_outer.___2677 + 1, ___2977 + 1) == m_localProcess)
{ partitionIJKInfoMap[___2977] = m_outer.___2680.getIJKZonePartitionInfo(m_outer.___2677, ___2977); ___1844 partitionMinIJK; ___1844 partitionMaxIJK; m_outer.getPartitionExtentsWithoutGhostNodes(___2977, partitionMinIJK, partitionMaxIJK); m_outer.m_partitionMinNodeNumbers[___2977] = static_cast<uint64_t>(zoneSize.offsetAtIJK(partitionMinIJK)); m_outer.m_partitionMaxNodeNumbers[___2977] = static_cast<uint64_t>(zoneSize.offsetAtIJK(partitionMaxIJK)); } } exchangeGhostInfo(partitionIJKInfoMap); for (___2090::___2980 ___2977 = 0; ___2977 < numPartitions; ++___2977) { if (m_outer.___2337.zonePartitionGetOwnerProcess(m_outer.___2677 + 1, ___2977 + 1) == m_localProcess) m_outer.m_partitionWriters[___2977] = boost::make_shared<SZLOrderedPartitionWriter>( boost::ref(m_outer.m_varIter), m_outer.___2677, m_outer.m_baseZone, ___2977, boost::ref(m_outer.m_writeVariables), m_outer.m_writeConnectivity, boost::ref(m_outer.m_partitionTecUtil), partitionIJKInfoMap[___2977]); } } uint64_t SZLOrderedPartitionedZoneWriterMPI::zoneDataFileSize(bool ___2002) { uint64_t ___3358 = 0; MPICommunicator communicator(m_impl->m_comm); if (m_impl->m_localProcess == m_impl->m_mainProcess) { for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) { uint64_t partitionFileSize; int32_t partitionOwner = ___2337.zonePartitionGetOwnerProcess(___2677 + 1, ___2977 + 1); if (partitionOwner == m_impl->m_localProcess) partitionFileSize = m_partitionWriters[___2977]->zoneFileSize(___2002); else communicator.receiveScalar(partitionFileSize, partitionOwner, TecioMPI::PARTITION_FILE_SIZE); ___3358 += partitionFileSize; } } else { for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) { int32_t partitionOwner = ___2337.zonePartitionGetOwnerProcess(___2677 + 1, ___2977 + 1); if (partitionOwner == m_impl->m_localProcess) { uint64_t partitionFileSize = m_partitionWriters[___2977]->zoneFileSize(___2002); communicator.sendScalar(partitionFileSize, m_impl->m_mainProcess, TecioMPI::PARTITION_FILE_SIZE); } } } return ___3358; } uint64_t SZLOrderedPartitionedZoneWriterMPI::zoneHeaderFileSize(bool ___2002) { if (m_impl->m_localProcess == m_impl->m_mainProcess) return SZLOrderedPartitionedZoneWriter::zoneHeaderFileSize(___2002); else return 0; } ___372 SZLOrderedPartitionedZoneWriterMPI::writeZoneData(___3933::FileWriterInterface& szpltFile) { ___1393 fileLoc = szpltFile.fileLoc(); boost::unordered_map<___4636, ___1393> partitionFileLocMap; uint64_t localPartitionFileSize = 0; if (m_impl->m_useCollectiveRoutines) { ___478(m_partitionWriters.size() == 1); ___4636 localPartition = m_partitionWriters.begin()->first; localPartitionFileSize = m_partitionWriters[localPartition]->zoneFileSize(szpltFile.___2002() == ___4226); gatherScatterPartitionFileLocs(fileLoc, partitionFileLocMap[localPartition], ___2337, ___2677, m_impl->m_localProcess, localPartitionFileSize, m_impl->m_mainProcess, m_impl->m_comm); } else if (m_impl->m_localProcess == m_impl->m_mainProcess) { MPICommunicator communicator(m_impl->m_comm); for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) { uint64_t partitionFileSize; int32_t partitionOwner = ___2337.zonePartitionGetOwnerProcess(___2677 + 1, ___2977 + 1); if (partitionOwner == m_impl->m_localProcess) { partitionFileLocMap[___2977] = fileLoc; partitionFileSize = m_partitionWriters[___2977]->zoneFileSize(szpltFile.___2002() == ___4226); } else { communicator.sendScalar(fileLoc, partitionOwner, TecioMPI::PARTITION_FILE_LOC); communicator.receiveScalar(partitionFileSize, partitionOwner, TecioMPI::PARTITION_FILE_SIZE); } fileLoc += partitionFileSize; } } else { MPINonBlockingCommunicationCollection communicationCollection(m_impl->m_comm); for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) { if (___2337.zonePartitionGetOwnerProcess(___2677 + 1, ___2977 + 1) == m_impl->m_localProcess) { uint64_t partitionSize = m_partitionWriters[___2977]->zoneFileSize(szpltFile.___2002() == ___4226); communicationCollection.sendScalarCopy(partitionSize, m_impl->m_mainProcess, TecioMPI::PARTITION_FILE_SIZE); communicationCollection.receiveScalar(partitionFileLocMap[___2977], m_impl->m_mainProcess, TecioMPI::PARTITION_FILE_LOC); } } communicationCollection.___4446(); } if (m_impl->m_useCollectiveRoutines) { ___478(m_partitionWriters.size() == 1 && localPartitionFileSize > 0); ___4636 localPartition = m_partitionWriters.begin()->first; MPIFileWriter& fileWriter = dynamic_cast<MPIFileWriter&>(szpltFile); boost::scoped_ptr<MPIFileWriter::ScopedCaching> scopedCaching; try { scopedCaching.reset(new MPIFileWriter::ScopedCaching(fileWriter, localPartitionFileSize)); } catch (...) { } m_partitionWriters[localPartition]->writeZone(szpltFile, partitionFileLocMap[localPartition]); m_partitionHeaderFilePositions[localPartition] = m_partitionWriters[localPartition]->getZoneHeaderFilePosition();
MPICommunicationCache cache; cache.addScalar(m_partitionHeaderFilePositions[localPartition], TecioMPI::PARTITION_HEADER_FILE_LOC); cache.addScalar(m_partitionMinNodeNumbers[localPartition], TecioMPI::PARTITION_MIN_NODE_NUMBER); cache.addScalar(m_partitionMaxNodeNumbers[localPartition], TecioMPI::PARTITION_MAX_NODE_NUMBER); SimpleVector<___2479> minMaxValues(m_varIter.___2812()); m_varIter.reset(); int i = 0; while (m_varIter.hasNext()) { ___4352 const datasetVar = m_varIter.next(); minMaxValues[i++] = m_partitionWriters[localPartition]->varMinMax(datasetVar); } cache.addVector(minMaxValues, TecioMPI::PARTITION_VAR_MINMAX_VALUES); SimpleVector<uint8_t> cachedData = cache.data(); int bytesPerPartition = cachedData.size(); std::vector<uint8_t> serializedCaches(static_cast<size_t>(bytesPerPartition) * ___2337.zoneGetNumPartitions(___2677 + 1)); MPI_Gather(&cachedData[0], bytesPerPartition, MPI_UINT8_T, &serializedCaches[0], bytesPerPartition, MPI_UINT8_T, m_impl->m_mainProcess, m_impl->m_comm); if (m_impl->m_localProcess == m_impl->m_mainProcess) { for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) { MPICommunicationCache cache(&serializedCaches[___2977 * bytesPerPartition], bytesPerPartition); int32_t partitionOwner = ___2337.zonePartitionGetOwnerProcess(___2677 + 1, ___2977 + 1); cache.retrieveScalar(m_partitionHeaderFilePositions[partitionOwner], TecioMPI::PARTITION_HEADER_FILE_LOC); cache.retrieveScalar(m_partitionMinNodeNumbers[partitionOwner], TecioMPI::PARTITION_MIN_NODE_NUMBER); cache.retrieveScalar(m_partitionMaxNodeNumbers[partitionOwner], TecioMPI::PARTITION_MAX_NODE_NUMBER); SimpleVector<___2479> minMaxValues; cache.retrieveVector(minMaxValues, TecioMPI::PARTITION_VAR_MINMAX_VALUES); ___478(minMaxValues.size() == m_varIter.___2812()); m_varIter.reset(); ___4352 const baseVar = m_varIter.baseItem(); while (m_varIter.hasNext()) { ___4352 const fileVar = m_varIter.next() - baseVar; m_varPartitionMinMaxes[fileVar][partitionOwner] = minMaxValues[fileVar]; } } } } else { for (___4636 ___2977 = 0; ___2977 < ___2337.zoneGetNumPartitions(___2677 + 1); ++___2977) { int32_t partitionOwner = ___2337.zonePartitionGetOwnerProcess(___2677 + 1, ___2977 + 1); if (partitionOwner == m_impl->m_localProcess) { m_partitionWriters[___2977]->writeZone(szpltFile, partitionFileLocMap[___2977]); m_partitionHeaderFilePositions[___2977] = m_partitionWriters[___2977]->getZoneHeaderFilePosition(); if (m_impl->m_localProcess == m_impl->m_mainProcess) { m_varIter.reset(); ___4352 const baseVar = m_varIter.baseItem(); while (m_varIter.hasNext()) { ___4352 const datasetVar = m_varIter.next(); m_varPartitionMinMaxes[datasetVar - baseVar][___2977] = m_partitionWriters[___2977]->varMinMax(datasetVar); } } else { MPICommunicationCache cache; cache.addScalar(m_partitionHeaderFilePositions[___2977], TecioMPI::PARTITION_HEADER_FILE_LOC); cache.addScalar(m_partitionMinNodeNumbers[___2977], TecioMPI::PARTITION_MIN_NODE_NUMBER); cache.addScalar(m_partitionMaxNodeNumbers[___2977], TecioMPI::PARTITION_MAX_NODE_NUMBER); SimpleVector<___2479> minMaxValues(m_varIter.___2812()); m_varIter.reset(); int i = 0; while (m_varIter.hasNext()) { ___4352 const datasetVar = m_varIter.next(); minMaxValues[i++] = m_partitionWriters[___2977]->varMinMax(datasetVar); } cache.addVector(minMaxValues, TecioMPI::PARTITION_VAR_MINMAX_VALUES); MPICommunicator(m_impl->m_comm).sendVector(cache.data(), m_impl->m_mainProcess, TecioMPI::PARTITION_CACHED_INFO_SIZE, TecioMPI::PARTITION_CACHED_INFO_VEC); } } else if (m_impl->m_localProcess == m_impl->m_mainProcess) { MPICommunicator communicator(m_impl->m_comm); SimpleVector<uint8_t> cachedData; communicator.receiveVector(cachedData, partitionOwner, TecioMPI::PARTITION_CACHED_INFO_SIZE, TecioMPI::PARTITION_CACHED_INFO_VEC); MPICommunicationCache cache(cachedData); cache.retrieveScalar(m_partitionHeaderFilePositions[___2977], TecioMPI::PARTITION_HEADER_FILE_LOC); cache.retrieveScalar(m_partitionMinNodeNumbers[___2977], TecioMPI::PARTITION_MIN_NODE_NUMBER); cache.retrieveScalar(m_partitionMaxNodeNumbers[___2977], TecioMPI::PARTITION_MAX_NODE_NUMBER); SimpleVector<___2479> minMaxValues; cache.retrieveVector(minMaxValues, TecioMPI::PARTITION_VAR_MINMAX_VALUES); ___478(minMaxValues.size() == m_varIter.___2812()); m_varIter.reset(); ___4352 const baseVar = m_varIter.baseItem(); while (m_varIter.hasNext()) { ___4352 const fileVar = m_varIter.next() - baseVar; m_varPartitionMinMaxes[fileVar][___2977] = minMaxValues[fileVar]; } } } } if (m_impl->m_localProcess == m_impl->m_mainProcess) szpltFile.___3459(fileLoc);
 #if !defined NO_ASSERTS
if (m_impl->m_localProcess != m_impl->m_mainProcess) szpltFile.___3459(0);
 #endif
return ___4226; } ___372 SZLOrderedPartitionedZoneWriterMPI::writeZoneHeader(___3933::FileWriterInterface& szpltFile) { if (m_impl->m_localProcess == m_impl->m_mainProcess) return SZLOrderedPartitionedZoneWriter::writeZoneHeader(szpltFile); else return ___4226; } } }
