This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via a51b0f04291f75748f4e929ef364c2157abfb67b (commit) from dd214fe7e3e2ffeaa2a81d605e724bca026dfc61 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/a51b0f04291f75748f4e929ef364c2157a... commit a51b0f04291f75748f4e929ef364c2157abfb67b Author: Rob Latham <[email protected]> Date: Wed Aug 21 13:55:59 2013 +0000 Improve aggregation selection on Blue Gene The default "aggregators per peset" value of 8 had not been changed since Blue Gene /L. Further, allow one to set the environment variable BGMPIO_NAGG_PSET to have the same effect as setting the "bg_nodes_pset" hint. Since this is all very confusing on /Q, toss in a few more (but no doubt not enough) comments. Reviewed-by: Bob Cernohous <[email protected]> diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_hints.c b/src/mpi/romio/adio/ad_bg/ad_bg_hints.c index bd3bfa6..22b5e37 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_hints.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_hints.c @@ -76,6 +76,7 @@ void ADIOI_BG_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code) */ if (!fd->hints->initialized) { + ad_bg_get_env_vars(); did_anything = 1; /* buffer size for collective I/O */ diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_pset.c b/src/mpi/romio/adio/ad_bg/ad_bg_pset.c index b5d9026..5eaf1c1 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_pset.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_pset.c @@ -254,7 +254,7 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, conf->nAggrs = n_aggrs; /* First pass gets nAggrs = -1 */ if(conf->nAggrs <=0) - conf->nAggrs = ADIOI_BG_NAGG_PSET_DFLT; + conf->nAggrs = bgmpio_bg_nagg_pset; if(conf->ioMinSize <= conf->nAggrs) conf->nAggrs = MAX(1,conf->ioMinSize-1); /* not including bridge itself */ /* if(conf->nAggrs > conf->numBridgeRanks) diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_pset.h b/src/mpi/romio/adio/ad_bg/ad_bg_pset.h index ba881d4..ab57e46 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_pset.h +++ b/src/mpi/romio/adio/ad_bg/ad_bg_pset.h @@ -51,12 +51,6 @@ typedef struct { #define MIN(a,b) ((a<b ? a : b)) -/* Default is to choose 8 aggregator nodes in each 32 CN pset. - Also defines default ratio of aggregator nodes in each a pset. - For Virtual Node Mode, the ratio is 8/64 */ -#define ADIOI_BG_NAGG_PSET_MIN 1 -#define ADIOI_BG_NAGG_PSET_DFLT 8 -#define ADIOI_BG_PSET_SIZE_DFLT 32 /* public funcs for ADIOI_BG_ProcInfo_t objects */ diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_tuning.c b/src/mpi/romio/adio/ad_bg/ad_bg_tuning.c index ec670c4..632bacd 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_tuning.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_tuning.c @@ -33,6 +33,7 @@ int bgmpio_comm; int bgmpio_tunegather; int bgmpio_tuneblocking; long bglocklessmpio_f_type; +int bgmpio_bg_nagg_pset; double bgmpio_prof_cw [BGMPIO_CIO_LAST]; double bgmpio_prof_cr [BGMPIO_CIO_LAST]; @@ -80,6 +81,11 @@ double bgmpio_prof_cr [BGMPIO_CIO_LAST]; * statfs() field f_type. * - The default is 0x20030528 (PVFS2_SUPER_MAGIC) * + * - BGMPIO_NAGG_PSET - Specify a ratio of "I/O aggregators" to use for each + * compute group (compute nodes + i/o nodes). Possible values: + * - any integer + * - Default is 8 + * */ void ad_bg_get_env_vars() { char *x, *dummy; @@ -104,6 +110,12 @@ void ad_bg_get_env_vars() { if (x) bglocklessmpio_f_type = strtol(x,&dummy,0); DBG_FPRINTF(stderr,"BGLOCKLESSMPIO_F_TYPE=%ld/%#lX\n", bglocklessmpio_f_type,bglocklessmpio_f_type); + /* note: this value will be 'sanity checked' in ADIOI_BG_persInfo_init(), + * when we know a bit more about what "largest possible value" and + * "smallest possible value" should be */ + bgmpio_bg_nagg_pset = ADIOI_BG_NAGG_PSET_DFLT; + x = getenv("BGMPIO_NAGG_PSET"); + if (x) bgmpio_bg_nagg_pset = atoi(x); } /* report timing breakdown for MPI I/O collective call */ diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_tuning.h b/src/mpi/romio/adio/ad_bg/ad_bg_tuning.h index e8e90ee..98cca71 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_tuning.h +++ b/src/mpi/romio/adio/ad_bg/ad_bg_tuning.h @@ -62,6 +62,19 @@ extern int bgmpio_tunegather; extern int bgmpio_tuneblocking; extern long bglocklessmpio_f_type; +/* Default is, well, kind of complicated. Blue Gene /L and /P had "psets": one + * i/o node and all compute nodes wired to it. On Blue Gene /Q that + * relationship is a lot more fluid. There are still I/O nodes, and compute + * nodes are assigned to an i/o node, but there are two routes to the i/o node, + * via compute nodes designated as "bridge nodes". In this code, what we used + * to call a "pset" is actually "compute nodes associated with and including a + * bridge node". So, "nAgg" is roughly "number of aggregators per bridge", but + * look closely at ADIOI_BG_persInfo_init() for the details */ + +#define ADIOI_BG_NAGG_PSET_DFLT 64 + +extern int bgmpio_bg_nagg_pset; + /* set internal variables for tuning environment variables */ void ad_bg_get_env_vars(); ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_bg/ad_bg_hints.c | 1 + src/mpi/romio/adio/ad_bg/ad_bg_pset.c | 2 +- src/mpi/romio/adio/ad_bg/ad_bg_pset.h | 6 ------ src/mpi/romio/adio/ad_bg/ad_bg_tuning.c | 12 ++++++++++++ src/mpi/romio/adio/ad_bg/ad_bg_tuning.h | 13 +++++++++++++ 5 files changed, 27 insertions(+), 7 deletions(-) hooks/post-receive -- MPICH primary repository