This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 9b51441012556210417c6301348e38b32dc3c35e (commit) via dacc232decdea8a0ba2c9576b59a794120b8283c (commit) from 2d2ff83d6be5c168e908a4cd426beea238a86885 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/9b51441012556210417c6301348e38b32d... commit 9b51441012556210417c6301348e38b32dc3c35e Author: Paul Coffman <[email protected]> Date: Sun Aug 31 22:14:35 2014 -0500 Additional optimization of gpfsmpio_bridgeringagg during MPI_File_open The code that chose the aggs for gpfsmpio_bridgeringagg was written very inefficiently resulting in a large performance degradation during the MPI_File_open. That code has been rewritten to squash the five torus dimensions into one dimension and to loop over that new representation fewer times. Now there is no performance degradation. Signed-off-by: Rob Latham <[email protected]> diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c index 1141bab..240c013 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c @@ -167,6 +167,12 @@ typedef struct int bridge; } sortstruct; +typedef struct +{ + int bridgeRank; + int numAggsAssigned; +} bridgeAggAssignment; + static int intsort(const void *p1, const void *p2) { sortstruct *i1, *i2; @@ -215,91 +221,78 @@ ADIOI_BG_compute_agg_ranklist_serial_do (const ADIOI_BG_ConfInfo_t *confInfo, } else { // aggTotal > 1 - ADIOI_BG_ProcInfo_t *allProcInfoAggNodeList = (ADIOI_BG_ProcInfo_t *) ADIOI_Malloc(confInfo->nProcs * sizeof(ADIOI_BG_ProcInfo_t)); - int allProcInfoAggNodeListSize = 0; - int maxManhattanDistanceToBridge = 0; - - // for ppn > 1, assign minumum rank as agg candidate - for (i=0;i<confInfo->nProcs;i++) { - int addProcToAggNodeList = 1; - for (j=0;j<allProcInfoAggNodeListSize;j++) { - if ((allProcInfoAggNodeList[j].torusCoords[0] == all_procInfo[i].torusCoords[0]) && - (allProcInfoAggNodeList[j].torusCoords[1] == all_procInfo[i].torusCoords[1]) && - (allProcInfoAggNodeList[j].torusCoords[2] == all_procInfo[i].torusCoords[2]) && - (allProcInfoAggNodeList[j].torusCoords[3] == all_procInfo[i].torusCoords[3]) && - (allProcInfoAggNodeList[j].torusCoords[4] == all_procInfo[i].torusCoords[4]) && - addProcToAggNodeList) { - // proc is in the node list, replace if this rank is smaller - addProcToAggNodeList = 0; - - if (allProcInfoAggNodeList[j].rank > all_procInfo[i].rank) - allProcInfoAggNodeList[j] = all_procInfo[i]; - } - } // for j - if (addProcToAggNodeList) { - allProcInfoAggNodeList[allProcInfoAggNodeListSize] = all_procInfo[i]; - if (allProcInfoAggNodeList[allProcInfoAggNodeListSize].manhattanDistanceToBridge > maxManhattanDistanceToBridge) - maxManhattanDistanceToBridge = allProcInfoAggNodeList[allProcInfoAggNodeListSize].manhattanDistanceToBridge; - allProcInfoAggNodeListSize++; - } - } // for i + int currentAggListSize = 0; + int numBridgesWithAggAssignments = 0; + bridgeAggAssignment *aggAssignments = (bridgeAggAssignment *)ADIOI_Malloc(confInfo->numBridgeRanks * sizeof(bridgeAggAssignment)); -#ifdef bridgeringaggtrace - fprintf(stderr,"allProcInfoAggNodeListSize is %d aggTotal is %d\n",allProcInfoAggNodeListSize,aggTotal); -#endif - - int *aggNodeBridgeList = (int *) ADIOI_Malloc (allProcInfoAggNodeListSize * sizeof(int)); // list of all bridge ranks - int *aggNodeBridgeListNum = (int *) ADIOI_Malloc (allProcInfoAggNodeListSize * sizeof(int)); - for (i=0;i<allProcInfoAggNodeListSize;i++) { - aggNodeBridgeList[i] = -1; - aggNodeBridgeListNum[i] = 0; - } - - int aggNodeBridgeListSize = 0; - for (i=0;i<allProcInfoAggNodeListSize;i++) { - int foundBridge = 0; - for (j=0;(j<aggNodeBridgeListSize && !foundBridge);j++) { - if (aggNodeBridgeList[j] == allProcInfoAggNodeList[i].bridgeRank) { - foundBridge = 1; - aggNodeBridgeListNum[i]++; - } - } - if (!foundBridge) { - aggNodeBridgeList[aggNodeBridgeListSize] = allProcInfoAggNodeList[i].bridgeRank; - aggNodeBridgeListNum[aggNodeBridgeListSize] = 1; - aggNodeBridgeListSize++; - } - } + int partitionSize = all_procInfo[0].numNodesInPartition; + int *nodesAssigned = (int *)ADIOI_Malloc(partitionSize * sizeof(int)); + for (i=0;i<partitionSize;i++) + nodesAssigned[i] = 0; - // add aggs based on numAggs per bridge, starting at gpfsmpio_bridgeringagg hops and increasing until numAggs aggs found - int currentAggListSize = 0; - for (i=0;i<aggNodeBridgeListSize;i++) { - int currentBridge = aggNodeBridgeList[i]; int currentNumHops = gpfsmpio_bridgeringagg; - int numAggsAssignedToThisBridge = 0; - while ((numAggsAssignedToThisBridge < numAggs) && (currentNumHops <= maxManhattanDistanceToBridge)) { - for (j=0;j<allProcInfoAggNodeListSize;j++) { - if (allProcInfoAggNodeList[j].bridgeRank == currentBridge) { - if (allProcInfoAggNodeList[j].manhattanDistanceToBridge == currentNumHops) { - aggList[currentAggListSize] = allProcInfoAggNodeList[j].rank; + int allAggsAssigned = 0; + + /* Iterate thru the process infos and select aggregators starting at currentNumHops + away. Increase the currentNumHops until all bridges have numAggs assigned to them. + */ + while (!allAggsAssigned) { + /* track whether any aggs are selected durng this round */ + int startingCurrentAggListSize = currentAggListSize; + int numIterForHopsWithNoAggs = 0; + for (i=0;i<confInfo->nProcs;i++) { + if (all_procInfo[i].manhattanDistanceToBridge == currentNumHops) { + if (nodesAssigned[all_procInfo[i].nodeRank] == 0) { // node is not assigned as an agg yet + int foundBridge = 0; + for (j=0;(j<numBridgesWithAggAssignments && !foundBridge);j++) { + if (aggAssignments[j].bridgeRank == all_procInfo[i].bridgeRank) { + foundBridge = 1; + if (aggAssignments[j].numAggsAssigned < numAggs) { + aggAssignments[j].numAggsAssigned++; + nodesAssigned[all_procInfo[i].nodeRank] = 1; + aggList[currentAggListSize] = all_procInfo[i].rank; + currentAggListSize++; #ifdef bridgeringaggtrace - printf("Assigned agg rank %d at torus coords %u %u %u %u %u to bridge %d at torus coords %u %u %u %u %u at a distance of %d hops\n",allProcInfoAggNodeList[j].rank,allProcInfoAggNodeList[j].torusCoords[0],allProcInfoAggNodeList[j].torusCoords[1],allProcInfoAggNodeList[j].torusCoords[2],allProcInfoAggNodeList[j].torusCoords[3],allProcInfoAggNodeList[j].torusCoords[4], currentBridge, all_procInfo[currentBridge].torusCoords[0], all_procInfo[currentBridge].torusCoords[1], all_procInfo[currentBridge].torusCoords[2], all_procInfo[currentBridge].torusCoords[3], all_procInfo[currentBridge].torusCoords[4],currentNumHops); + printf("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",all_procInfo[i].rank,all_procInfo[i].nodeRank,all_procInfo[i].bridgeRank,currentNumHops); #endif + } + } + } + if (!foundBridge) { + aggAssignments[numBridgesWithAggAssignments].bridgeRank = all_procInfo[i].bridgeRank; + aggAssignments[numBridgesWithAggAssignments].numAggsAssigned = 1; + numBridgesWithAggAssignments++; + nodesAssigned[all_procInfo[i].nodeRank] = 1; + aggList[currentAggListSize] = all_procInfo[i].rank; currentAggListSize++; - numAggsAssignedToThisBridge++; - if (numAggsAssignedToThisBridge >= numAggs) - break; +#ifdef bridgeringaggtrace + printf("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",all_procInfo[i].rank,all_procInfo[i].nodeRank,all_procInfo[i].bridgeRank,currentNumHops); +#endif } } } - currentNumHops++; - } // while - ADIOI_Assert(numAggsAssignedToThisBridge == numAggs); - } // for - - ADIOI_Free(allProcInfoAggNodeList); - ADIOI_Free(aggNodeBridgeList); - ADIOI_Free(aggNodeBridgeListNum); + } + + if (numBridgesWithAggAssignments == confInfo->numBridgeRanks) { + allAggsAssigned = 1; + for (i=0;(i<numBridgesWithAggAssignments && allAggsAssigned);i++) { + if (aggAssignments[i].numAggsAssigned < numAggs) + allAggsAssigned = 0; + } + } + currentNumHops++; + /* If 3 rounds go by without selecting an agg abort to avoid + infinite loop. + */ + if (startingCurrentAggListSize == currentAggListSize) + numIterForHopsWithNoAggs++; + else + numIterForHopsWithNoAggs = 0; + ADIOI_Assert(numIterForHopsWithNoAggs <= 3); + } + + ADIOI_Free(aggAssignments); + ADIOI_Free(nodesAssigned); } // else aggTotal > 1 diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c index 59f2219..a18b02a 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c @@ -136,16 +136,23 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, proc->coreID = hw.coreID; if (gpfsmpio_bridgeringagg > 0) { - for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { - proc->torusCoords[i] = hw.Coords[i]; - } #ifdef bridgeringaggtrace if (rank == 0) fprintf(stderr,"Block dimensions:\n"); #endif + + /* Set the numNodesInPartition and nodeRank for this proc + */ + proc->numNodesInPartition = 1; + proc->nodeRank = 0; for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { torusSize[i] = hw.Size[i]; dimTorus[i] = hw.isTorus[i]; + proc->numNodesInPartition *= hw.Size[i]; + int baseNum = 1; + for (int j=0;j<i;j++) + baseNum *= hw.Size[j]; + proc->nodeRank += (hw.Coords[i] * baseNum); #ifdef bridgeringaggtrace if (rank == 0) fprintf(stderr,"Dimension %d has %d elements wrap-around value is %d\n",i,torusSize[i],dimTorus[i]); diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h index d5f36b1..bcdc61f 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h @@ -35,7 +35,8 @@ typedef struct { node, i.e. psetsize*/ int iamBridge; /* am *I* the bridge rank? */ int __ipad[2]; - unsigned torusCoords[MPIX_TORUS_MAX_DIMS]; /* torus coordinates of node on which this rank resides */ + unsigned nodeRank; /* torus coords converted to an integer for use with gpfsmpio_bridgeringagg */ + unsigned numNodesInPartition; /* number of physical nodes in the job partition */ unsigned manhattanDistanceToBridge; /* number of hops between this rank and the bridge node */ } ADIOI_BG_ProcInfo_t __attribute__((aligned(16))); http://git.mpich.org/mpich.git/commitdiff/dacc232decdea8a0ba2c9576b59a794120... commit dacc232decdea8a0ba2c9576b59a794120b8283c Author: Rob Latham <[email protected]> Date: Wed Sep 3 11:48:33 2014 -0500 fix compiler warnings and shadowed declarations We do not often enough compile with --enable-strict on blue gene No Reviewer diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c index 31ba04b..f4fef37 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c @@ -115,14 +115,14 @@ void ADIOI_GPFS_Open(ADIO_File fd, int *error_code) MPI_Comm_rank(fd->comm, &rank); if ((rank == fd->hints->ranklist[0]) || (fd->comm == MPI_COMM_SELF)) { - struct stat64 gpfs_stat; + struct stat64 gpfs_statbuf; /* Get the (real) underlying file system block size */ - rc = stat64(fd->filename, &gpfs_stat); + rc = stat64(fd->filename, &gpfs_statbuf); if (rc >= 0) { - fd->blksize = gpfs_stat.st_blksize; + fd->blksize = gpfs_statbuf.st_blksize; DBGV_FPRINTF(stderr,"Successful stat '%s'. Blocksize=%ld\n", - fd->filename,gpfs_stat.st_blksize); + fd->filename,gpfs_statbuf.st_blksize); } else { diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index d049340..0e2a1d2 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -572,7 +572,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype } } - ADIO_Offset st_loc_ion, end_loc_ion, needs_gpfs_access_cleanup=0; + ADIO_Offset st_loc_ion=0, end_loc_ion=0, needs_gpfs_access_cleanup=0; #ifdef BGQPLATFORM if (ntimes > 0) { /* only set the gpfs hint if we have io - ie this rank is an aggregator -- otherwise will fail for deferred open */ ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c | 8 +- src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c | 2 +- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c | 145 +++++++++++++-------------- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c | 13 ++- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h | 3 +- 5 files changed, 86 insertions(+), 85 deletions(-) hooks/post-receive -- MPICH primary repository