This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 38aacd18339d13666f8e36de5a7c109929aac0ac (commit) via f571b0e526ebbddb32802e7535c1472b0cd9499a (commit) via cc67b4001b7525fea08ebc8e3a91181341b47d05 (commit) from 1ecfc4ffbe8e45f6e449a555520c7b4eb31c9347 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/38aacd18339d13666f8e36de5a7c109929... commit 38aacd18339d13666f8e36de5a7c109929aac0ac Author: Rob Latham <[email protected]> Date: Wed Dec 4 12:13:11 2013 -0600 deferred open: fixup immediate-mode communicators If user lies about "romio_no_indep_rw" (does independent I/O anyway), ROMIO will open the file at the time of the read/write operation. this open call (Blue Gene, maybe others) might try to be clever, as when Blue Gene does a stat-and-broadcast. Since this open is now an independent open, not collective, pass in COMM_SELF Signed-off-by: Ken Raffenetti <[email protected]> diff --git a/src/mpi/romio/adio/common/ad_iopen.c b/src/mpi/romio/adio/common/ad_iopen.c index b678541..e999df8 100644 --- a/src/mpi/romio/adio/common/ad_iopen.c +++ b/src/mpi/romio/adio/common/ad_iopen.c @@ -10,12 +10,13 @@ void ADIO_ImmediateOpen(ADIO_File fd, int *error_code) { int nprocs, myrank; + MPI_Comm tmp_comm; + tmp_comm = fd->comm; + /* some file systems might try to be clever inside their open routine. + * e.g. Blue Gene does a stat-and-broadcast */ + fd->comm = MPI_COMM_SELF; (*(fd->fns->ADIOI_xxx_Open))(fd, error_code); fd->is_open = 1; + fd->comm = tmp_comm; - /* DEBUG: remove following lines */ - MPI_Comm_size(fd->comm, &nprocs); - MPI_Comm_rank(fd->comm, &myrank); - FPRINTF(stdout, "[%d/%d] DEBUG: ADIOI_ImmediateOpen called on %s\n", - myrank, nprocs, fd->filename); } http://git.mpich.org/mpich.git/commitdiff/f571b0e526ebbddb32802e7535c1472b0c... commit f571b0e526ebbddb32802e7535c1472b0cd9499a Author: Rob Latham <[email protected]> Date: Wed Dec 4 11:59:20 2013 -0600 Introduce and use ADIO_TWO_PHASE feature-check the old deferred open code explicitly checked for the presence of ADIOI_GEN_WriteStrided and ADIOI_GEN_ReadStrided functions in the file system specific struct. This is needlessly specific: lustre and bluegene, for example re-implemented two phase but as a different (and differently named) function Signed-off-by: Ken Raffenetti <[email protected]> diff --git a/src/mpi/romio/adio/common/ad_features.c b/src/mpi/romio/adio/common/ad_features.c index 339c54d..5be6cb3 100644 --- a/src/mpi/romio/adio/common/ad_features.c +++ b/src/mpi/romio/adio/common/ad_features.c @@ -8,6 +8,7 @@ int ADIOI_GEN_Feature(ADIO_File fd, int flag) case ADIO_ATOMIC_MODE: case ADIO_DATA_SIEVING_WRITES: case ADIO_UNLINK_AFTER_CLOSE: + case ADIO_TWO_PHASE: return 1; break; case ADIO_SCALABLE_OPEN: diff --git a/src/mpi/romio/adio/common/ad_open.c b/src/mpi/romio/adio/common/ad_open.c index cfd1e7a..8a9d33b 100644 --- a/src/mpi/romio/adio/common/ad_open.c +++ b/src/mpi/romio/adio/common/ad_open.c @@ -215,28 +215,21 @@ int is_aggregator(int rank, ADIO_File fd ) { return 0; } -/* - * we special-case TESTFS because all it does is wrap logging info around GEN +/* + * If file system implements some version of two-phase -- doesn't have to be + * generic -- we can still carry out the defered open optimization */ static int uses_generic_read(ADIO_File fd) { - ADIOI_Fns *fns = fd->fns; - if (fns->ADIOI_xxx_ReadStridedColl == ADIOI_GEN_ReadStridedColl || - fd->file_system == ADIO_TESTFS ) - { + if (ADIO_Feature(fd, ADIO_TWO_PHASE)) return 1; - } return 0; } static int uses_generic_write(ADIO_File fd) { - ADIOI_Fns *fns = fd->fns; - if (fns->ADIOI_xxx_WriteStridedColl == ADIOI_GEN_WriteStridedColl || - fd->file_system == ADIO_TESTFS ) - { + if (ADIO_Feature(fd, ADIO_TWO_PHASE)) return 1; - } return 0; } diff --git a/src/mpi/romio/adio/include/adio.h b/src/mpi/romio/adio/include/adio.h index 26f1659..f2e68b4 100644 --- a/src/mpi/romio/adio/include/adio.h +++ b/src/mpi/romio/adio/include/adio.h @@ -314,6 +314,9 @@ typedef struct { #define ADIO_UNLINK_AFTER_CLOSE 305 /* supports posix semantic of keeping a deleted file around until all processors have closed it */ +#define ADIO_TWO_PHASE 306 /* file system implements some version of + two-phase collective buffering with + aggregation */ /* for default file permissions */ #define ADIO_PERM_NULL -1 http://git.mpich.org/mpich.git/commitdiff/cc67b4001b7525fea08ebc8e3a91181341... commit cc67b4001b7525fea08ebc8e3a91181341b47d05 Author: Rob Latham <[email protected]> Date: Mon Dec 2 16:45:30 2013 -0600 re-enable deferred open on Blue Gene deferred open had been disabled, but it does not need to be. Thanks for drawing this to my attention, Paul Coffman <[email protected]>. Signed-off-by: Ken Raffenetti <[email protected]> diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_hints.c b/src/mpi/romio/adio/ad_bg/ad_bg_hints.c index 65aaea7..da86206 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_hints.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_hints.c @@ -274,12 +274,24 @@ void ADIOI_BG_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code) if (did_anything) { ADIOI_BG_gen_agg_ranklist(fd, fd->hints->cb_nodes); } - /* ignore defered open hints and do not enable it for bluegene: need all - * processors in the open path so we can stat-and-broadcast the blocksize - */ - ADIOI_Info_set(info, "romio_no_indep_rw", "false"); - fd->hints->no_indep_rw = 0; - fd->hints->deferred_open = 0; + + /* deferred_open won't be set by callers, but if the user doesn't + * explicitly disable collecitve buffering (two-phase) and does hint that + * io w/o independent io is going on, we'll set this internal hint as a + * convenience */ + if ( ( (fd->hints->cb_read != ADIOI_HINT_DISABLE) \ + && (fd->hints->cb_write != ADIOI_HINT_DISABLE)\ + && fd->hints->no_indep_rw ) ) { + fd->hints->deferred_open = 1; + } else { + /* setting romio_no_indep_rw enable and romio_cb_{read,write} + * disable at the same time doesn't make sense. honor + * romio_cb_{read,write} and force the no_indep_rw hint to + * 'disable' */ + ADIOI_Info_set(info, "romio_no_indep_rw", "false"); + fd->hints->no_indep_rw = 0; + fd->hints->deferred_open = 0; + } /* BobC commented this out, but since hint processing runs on both bg and * bglockless, we need to keep DS writes enabled on gpfs and disabled on diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_open.c b/src/mpi/romio/adio/ad_bg/ad_bg_open.c index 01f135f..bddcfaf 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_open.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_open.c @@ -116,7 +116,7 @@ static void scaleable_stat(ADIO_File fd) long buf[2]; MPI_Comm_rank(fd->comm, &rank); - if (rank == 0) { + if (rank == fd->hints->ranklist[0]) { /* Get the (real) underlying file system block size */ rc = stat64(fd->filename, &bg_stat); if (rc >= 0) @@ -161,7 +161,13 @@ static void scaleable_stat(ADIO_File fd) } } /* now we can broadcast the stat/statfs data to everyone else */ - MPI_Bcast(buf, 2, MPI_LONG, 0, fd->comm); + if (fd->comm != MPI_COMM_SELF) { /* if indep open, there's no one to talk to*/ + if (fd->agg_comm != MPI_COMM_NULL) /* deferred open: only a subset of + processes participate */ + MPI_Bcast(buf, 2, MPI_LONG, 0, fd->agg_comm); + else + MPI_Bcast(buf, 2, MPI_LONG, 0, fd->comm); + } bg_stat.st_blksize = buf[0]; bg_statfs.f_type = buf[1]; @@ -178,7 +184,7 @@ static void scaleable_stat(ADIO_File fd) /* Only one rank is an "fsync aggregator" because only one * fsync is needed */ - if (rank == 0) + if (rank == fd->hints->ranklist[0]) { ((ADIOI_BG_fs*)fd->fs_ptr)->fsync_aggr |= ADIOI_BG_FSYNC_AGGREGATOR; ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_bg/ad_bg_hints.c | 24 ++++++++++++++++++------ src/mpi/romio/adio/ad_bg/ad_bg_open.c | 12 +++++++++--- src/mpi/romio/adio/common/ad_features.c | 1 + src/mpi/romio/adio/common/ad_iopen.c | 11 ++++++----- src/mpi/romio/adio/common/ad_open.c | 17 +++++------------ src/mpi/romio/adio/include/adio.h | 3 +++ 6 files changed, 42 insertions(+), 26 deletions(-) hooks/post-receive -- MPICH primary repository