January 2014 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-156-g7b6824b
by noreply＠mpich.org 11 Jan '14

11 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 7b6824bf2e565f30bd147a3f72a9c6c66075615a (commit) from 5fd8fc67f63a8c9a6134d069516f90e234339f67 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/7b6824bf2e565f30bd147a3f72a9c6c66… commit 7b6824bf2e565f30bd147a3f72a9c6c66075615a Author: Rob Latham <robl(a)mcs.anl.gov> Date: Sat Jan 11 16:01:05 2014 -0600 release memory in error path Clean up memory allocated in write (for read-modify-write) when encountering error. Still todo: blue gene, lustre, and any other driver that acts this way. closes #1994 Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_nfs/ad_nfs_write.c b/src/mpi/romio/adio/ad_nfs/ad_nfs_write.c index 6b607a1..dda2fea 100644 --- a/src/mpi/romio/adio/ad_nfs/ad_nfs_write.c +++ b/src/mpi/romio/adio/ad_nfs/ad_nfs_write.c @@ -110,7 +110,7 @@ void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count, MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**ioRMWrdwr", 0); \ - return; \ + goto fn_exit; \ } \ } \ write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \ @@ -140,7 +140,7 @@ void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count, MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**ioRMWrdwr", 0); \ - return; \ + goto fn_exit; \ } \ write_sz = ADIOI_MIN(req_len, writebuf_len); \ memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\ @@ -164,7 +164,7 @@ void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count, MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**ioRMWrdwr", 0); \ - return; \ + goto fn_exit; \ } \ } \ write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \ @@ -186,7 +186,7 @@ void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count, MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**ioRMWrdwr", 0); \ - return; \ + goto fn_exit; \ } \ write_sz = ADIOI_MIN(req_len, writebuf_len); \ memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\ @@ -280,7 +280,7 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count, int buf_count, buftype_is_contig, filetype_is_contig; ADIO_Offset userbuf_off; ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off; - char *writebuf, *value; + char *writebuf=NULL, *value; int st_fwr_size, st_n_filetypes, writebuf_len, write_sz; int new_bwr_size, new_fwr_size, err_flag=0, info_flag, max_bufsize; static char myname[] = "ADIOI_NFS_WRITESTRIDED"; @@ -364,8 +364,6 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count, if (fd->atomicity) ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1); - ADIOI_Free(writebuf); /* malloced in the buffered_write macro */ - if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off; if (err_flag) { *error_code = MPIO_Err_create_code(MPI_SUCCESS, @@ -517,8 +515,8 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count, myname, __LINE__, MPI_ERR_IO, "ADIOI_NFS_WriteStrided: ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR.", 0); - return; - } + goto fn_exit; + } if (buftype_is_contig && !filetype_is_contig) { @@ -653,8 +651,6 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count, if (err == -1) err_flag = 1; - ADIOI_Free(writebuf); /* malloced in the buffered_write macro */ - if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off; if (err_flag) { *error_code = MPIO_Err_create_code(MPI_SUCCESS, @@ -674,4 +670,8 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count, #endif if (!buftype_is_contig) ADIOI_Delete_flattened(datatype); +fn_exit: + if (writebuf != NULL) ADIOI_Free(writebuf); + + return; } diff --git a/src/mpi/romio/adio/common/ad_write_str.c b/src/mpi/romio/adio/common/ad_write_str.c index b09d6f5..839feca 100644 --- a/src/mpi/romio/adio/common/ad_write_str.c +++ b/src/mpi/romio/adio/common/ad_write_str.c @@ -20,7 +20,7 @@ MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**iowswc", 0); \ - return; \ + goto fn_exit; \ } \ } \ writebuf_off = req_off; \ @@ -33,7 +33,7 @@ MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**iowsrc", 0); \ - return; \ + goto fn_exit; \ } \ } \ write_sz = (unsigned) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \ @@ -48,7 +48,7 @@ MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**iowswc", 0); \ - return; \ + goto fn_exit; \ } \ req_len -= write_sz; \ userbuf_off += write_sz; \ @@ -62,7 +62,7 @@ MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**iowsrc", 0); \ - return; \ + goto fn_exit; \ } \ write_sz = ADIOI_MIN(req_len, writebuf_len); \ memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\ @@ -82,7 +82,7 @@ MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**iowswc", 0); \ - return; \ + goto fn_exit; \ } \ writebuf_off = req_off; \ writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\ @@ -98,7 +98,7 @@ MPIR_ERR_RECOVERABLE, myname, \ __LINE__, MPI_ERR_IO, \ "**iowswc", 0); \ - return; \ + goto fn_exit; \ } \ req_len -= write_sz; \ userbuf_off += write_sz; \ @@ -126,7 +126,7 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count, int buf_count, buftype_is_contig, filetype_is_contig; ADIO_Offset userbuf_off; ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off; - char *writebuf; + char *writebuf=NULL; unsigned writebuf_len, max_bufsize, write_sz; MPI_Aint bufsize; ADIO_Status status1; @@ -215,9 +215,7 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count, if (fd->atomicity) ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1); - if (*error_code != MPI_SUCCESS) return; - - ADIOI_Free(writebuf); + if (*error_code != MPI_SUCCESS) goto fn_exit; if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off; } @@ -312,7 +310,7 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count, #ifdef HAVE_STATUS_SET_BYTES MPIR_Status_set_bytes(status, datatype, bufsize); #endif - return; + goto fn_exit; } /* Calculate end_offset, the last byte-offset that will be accessed. @@ -466,13 +464,11 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count, writebuf_off, &status1, error_code); if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); - if (*error_code != MPI_SUCCESS) return; + if (*error_code != MPI_SUCCESS) goto fn_exit; } if (fd->atomicity) ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1); - ADIOI_Free(writebuf); - if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off; } @@ -486,5 +482,7 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count, #endif if (!buftype_is_contig) ADIOI_Delete_flattened(datatype); +fn_exit: + if (writebuf != NULL) ADIOI_Free(writebuf); } ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_nfs/ad_nfs_write.c | 22 +++++++++++----------- src/mpi/romio/adio/common/ad_write_str.c | 26 ++++++++++++-------------- 2 files changed, 23 insertions(+), 25 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-155-g5fd8fc6
by noreply＠mpich.org 10 Jan '14

10 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 5fd8fc67f63a8c9a6134d069516f90e234339f67 (commit) via 4fc5c2057dabfe13cb345a61c6188ad7974ada36 (commit) from 2ce5a7e2786fd07ffb13dc4650fc231b7e9ef829 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/5fd8fc67f63a8c9a6134d069516f90e23… commit 5fd8fc67f63a8c9a6134d069516f90e234339f67 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Wed Jan 8 16:22:08 2014 -0600 gcc 4.4.6 warning cleanup. Signed-off-by: Su Huang <suhuang(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c b/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c index 4c52993..5a23e40 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c +++ b/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c @@ -150,9 +150,9 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIU_Free(req); } else { - req->buffer = origin_addr + req->origin.dt.true_lb; + req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); req->user_buffer = result_addr + req->origin.dt.true_lb; - req->compare_buffer = compare_addr + req->origin.dt.true_lb; + req->compare_buffer = (void *) ((uintptr_t) compare_addr + req->origin.dt.true_lb); pami_result_t rc; pami_task_t task = MPID_VCR_GET_LPID(win->comm_ptr->vcr, target_rank); diff --git a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c index a99a19e..7c64145 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c +++ b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c @@ -342,7 +342,7 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, req->op = op; req->pami_datatype = pami_type; /* MPI_Fetch_and_op only supports predefined datatype */ - req->buffer = origin_addr + req->origin.dt.true_lb; + req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); req->user_buffer = result_addr + req->origin.dt.true_lb; pami_result_t rc; diff --git a/src/mpid/pamid/src/onesided/mpid_win_reqops.c b/src/mpid/pamid/src/onesided/mpid_win_reqops.c index 129d5c8..a9350c2 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_reqops.c +++ b/src/mpid/pamid/src/onesided/mpid_win_reqops.c @@ -163,7 +163,7 @@ MPID_Rget(void *origin_addr, #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) int -MPID_Raccumulate(const void *origin_addr, +MPID_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 8604bb5..85804b1 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -207,7 +207,7 @@ typedef struct /** \todo make sure that the extra fields are removed */ -typedef struct _mpidi_win_request +typedef struct MPIDI_Win_request { MPID_Win *win; MPIDI_Win_requesttype_t type; @@ -255,7 +255,7 @@ typedef struct _mpidi_win_request void *compare_buffer; /* anchor of compare buffer for compare and swap */ uint32_t buffer_free; void *buffer; - struct _mpidi_win_request *next; + struct MPIDI_Win_request *next; MPI_Op op; int result_num_contig; http://git.mpich.org/mpich.git/commitdiff/4fc5c2057dabfe13cb345a61c6188ad79… commit 4fc5c2057dabfe13cb345a61c6188ad7974ada36 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Wed Jan 8 16:26:42 2014 -0600 fix missing const for MPID_Accumulate and MPID_Put functions. Signed-off-by: Su Huang <suhuang(a)us.ibm.com> Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h index 5f1bfc8..bfbd450 100644 --- a/src/include/mpiimpl.h +++ b/src/include/mpiimpl.h @@ -3338,11 +3338,11 @@ int MPID_Win_create(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *, MPID_Win **); int MPID_Win_free(MPID_Win **); -int MPID_Put(void *, int, MPI_Datatype, int, MPI_Aint, int, +int MPID_Put(const void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, MPID_Win *); int MPID_Get(void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, MPID_Win *); -int MPID_Accumulate(void *, int, MPI_Datatype, int, MPI_Aint, int, +int MPID_Accumulate(const void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, MPI_Op, MPID_Win *); int MPID_Win_fence(int, MPID_Win *); diff --git a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c index 2d04091..e328d30 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c @@ -159,7 +159,7 @@ MPIDI_Accumulate(pami_context_t context, #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) int -MPID_Accumulate(void *origin_addr, +MPID_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, @@ -265,7 +265,7 @@ MPID_Accumulate(void *origin_addr, if (req->origin.dt.contig) { req->buffer_free = 0; - req->buffer = origin_addr + req->origin.dt.true_lb; + req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); } else { diff --git a/src/mpid/pamid/src/onesided/mpid_win_put.c b/src/mpid/pamid/src/onesided/mpid_win_put.c index 110f294..b98a99d 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_put.c +++ b/src/mpid/pamid/src/onesided/mpid_win_put.c @@ -216,7 +216,7 @@ MPIDI_Put_use_pami_put(pami_context_t context, MPIDI_Win_request * req,int *fr #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) int -MPID_Put(void *origin_addr, +MPID_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, @@ -303,7 +303,7 @@ MPID_Put(void *origin_addr, if (req->origin.dt.contig) { req->buffer_free = 0; - req->buffer = origin_addr + req->origin.dt.true_lb; + req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); } else { ----------------------------------------------------------------------- Summary of changes: src/include/mpiimpl.h | 4 ++-- src/mpid/pamid/src/onesided/mpid_win_accumulate.c | 4 ++-- .../pamid/src/onesided/mpid_win_compare_and_swap.c | 4 ++-- .../pamid/src/onesided/mpid_win_fetch_and_op.c | 2 +- src/mpid/pamid/src/onesided/mpid_win_put.c | 4 ++-- src/mpid/pamid/src/onesided/mpid_win_reqops.c | 2 +- src/mpid/pamid/src/onesided/mpidi_onesided.h | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-153-g2ce5a7e
by noreply＠mpich.org 10 Jan '14

10 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 2ce5a7e2786fd07ffb13dc4650fc231b7e9ef829 (commit) via 1a0c3dbc9d8055314613016d5c64f801d9b87219 (commit) via 99485af86e826a946fe038a77f0b98244526dc4d (commit) from 3b24d31851794830cb449b03ba09bfc6cf1168f0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/2ce5a7e2786fd07ffb13dc4650fc231b7… commit 2ce5a7e2786fd07ffb13dc4650fc231b7e9ef829 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Tue Jan 7 15:02:30 2014 -0600 Prioritize thread-multiple branch. Give a "likely" attribute to the branch that checks if an MPI process is initialized in THREAD_MULTIPLE or not. This should reduce the cost of the branch for applications that require THREAD_MULTIPLE. The per-object mode in the pamid branch is intended for the case where comm-threads are used. In this case, the application is mostly in THREAD_MULTIPLE mode, even if it didn't explicitly initialize it as such. The only time it'll not be in that mode is during initialization before the appropriate mutexes have been setup. See #1900. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpid/pamid/include/mpidi_thread.h b/src/mpid/pamid/include/mpidi_thread.h index eb31b24..dff875e 100644 --- a/src/mpid/pamid/include/mpidi_thread.h +++ b/src/mpid/pamid/include/mpidi_thread.h @@ -107,45 +107,45 @@ #elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT -#define MPIDI_CS_ENTER(m) \ - do { \ - MPIU_THREAD_CHECK_BEGIN \ - MPIDI_Mutex_acquire(m); \ - MPIU_THREAD_CHECK_END \ +#define MPIDI_CS_ENTER(m) \ + do { \ + if (likely(MPIR_ThreadInfo.isThreaded)) { \ + MPIDI_Mutex_acquire(m); \ + } \ } while (0) -#define MPIDI_CS_EXIT(m) \ - do { \ - MPIU_THREAD_CHECK_BEGIN \ - MPIDI_Mutex_sync(); \ - MPIDI_Mutex_release(m); \ - MPIU_THREAD_CHECK_END \ +#define MPIDI_CS_EXIT(m) \ + do { \ + if (likely(MPIR_ThreadInfo.isThreaded)) { \ + MPIDI_Mutex_sync(); \ + MPIDI_Mutex_release(m); \ + } \ } while (0) -#define MPIDI_CS_YIELD(m) \ - do { \ - MPIU_THREAD_CHECK_BEGIN \ - MPIDI_Mutex_sync(); \ - MPIDI_Mutex_release(m); \ - MPIDI_Mutex_acquire(m); \ - MPIU_THREAD_CHECK_END \ +#define MPIDI_CS_YIELD(m) \ + do { \ + if (likely(MPIR_ThreadInfo.isThreaded)) { \ + MPIDI_Mutex_sync(); \ + MPIDI_Mutex_release(m); \ + MPIDI_Mutex_acquire(m); \ + } \ } while (0) -#define MPIDI_CS_TRY(m) \ - do { \ - MPIU_THREAD_CHECK_BEGIN \ - MPIDI_Mutex_try_acquire(m); \ - MPIU_THREAD_CHECK_END \ +#define MPIDI_CS_TRY(m) \ + do { \ + if (likely(MPIR_ThreadInfo.isThreaded)) { \ + MPIDI_Mutex_try_acquire(m); \ + } \ } while (0) -#define MPIDI_CS_SCHED_YIELD(m) \ - do { \ - MPIU_THREAD_CHECK_BEGIN \ - MPIDI_Mutex_sync(); \ - MPIDI_Mutex_release(m); \ - sched_yield(); \ - MPIDI_Mutex_acquire(m); \ - MPIU_THREAD_CHECK_END \ +#define MPIDI_CS_SCHED_YIELD(m) \ + do { \ + if (likely(MPIR_ThreadInfo.isThreaded)) { \ + MPIDI_Mutex_sync(); \ + MPIDI_Mutex_release(m); \ + sched_yield(); \ + MPIDI_Mutex_acquire(m); \ + } \ } while (0) #define MPIU_THREAD_CS_ALLFUNC_ENTER(_context) http://git.mpich.org/mpich.git/commitdiff/1a0c3dbc9d8055314613016d5c64f801d… commit 1a0c3dbc9d8055314613016d5c64f801d9b87219 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Tue Jan 7 00:40:17 2014 -0600 We always need the runtime thread-safety check. Disabling runtime check for thread-safety is not a correct option. This would cause memory allocation and string manipulation routines to be unusable before the thread-safety level is set. Fixes #1900. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/configure.ac b/configure.ac index 3c27a6b..21fc951 100644 --- a/configure.ac +++ b/configure.ac @@ -486,9 +486,6 @@ AC_ARG_WITH(logging, AC_HELP_STRING([--with-logging=name], [Specify the logging library for MPICH]), [if test -z "$withval" ; then with_logging=rlog ; fi],with_logging=none) -dnl The default option needs to be defined in terms of a specific choice -dnl (runtime in this case). Note that the default choice is the same as -dnl runtime only for certain devices - not for every device. AC_ARG_ENABLE(threads, [ --enable-threads=level - Control the level of thread support in the MPICH implementation. The following levels @@ -496,14 +493,8 @@ AC_ARG_ENABLE(threads, single - No threads (MPI_THREAD_SINGLE) funneled - Only the main thread calls MPI (MPI_THREAD_FUNNELED) serialized - User serializes calls to MPI (MPI_THREAD_SERIALIZED) - runtime - The level of thread support is determined by - the arguments to MPI_Init_thread, with - MPI_THREAD_MULTIPLE available. The default option - for many communication devices. - multiple - Fully multi-threaded (MPI_THREAD_MULTIPLE) always. - DO NOT select this option. The option runtime is more - efficient and also supports thread_multiple. - (multiple aliased to runtime now) + multiple - Fully multi-threaded (MPI_THREAD_MULTIPLE) + runtime - Alias to "multiple" See also the --enable-thread-cs option for controlling the granularity of the concurrency inside of the library @@ -1278,7 +1269,10 @@ fi # # Threads must be supported by the device. First, set the default to # be the highest supported by the device +# "runtime" was an old (now deprecated) option; just map it to multiple +if test "$enable_threads" = "runtime" ; then enable_threads=multiple ; fi if test "$enable_threads" = "yes" ; then enable_threads=default ; fi +if test "$enable_threads" = "no" ; then enable_threads=single ; fi if test "$enable_threads" = default ; then # XXX DJG bug is here, PREREQ is not being used right now if test -n "$MPID_MAX_THREAD_LEVEL" ; then @@ -1286,7 +1280,7 @@ if test "$enable_threads" = default ; then MPI_THREAD_SINGLE) enable_threads=single ;; MPI_THREAD_FUNNELED) enable_threads=funneled ;; MPI_THREAD_SERIALIZED) enable_threads=serialized ;; - MPI_THREAD_MULTIPLE) enable_threads=runtime ;; + MPI_THREAD_MULTIPLE) enable_threads=multiple ;; *) AC_MSG_ERROR([Unrecognized thread level from device $MPID_MAX_THREAD_LEVEL]) ;; esac @@ -1295,18 +1289,6 @@ if test "$enable_threads" = default ; then fi fi -if test "$enable_threads" = "default" ; then - enable_threads=runtime -elif test "$enable_threads" = "no" ; then - enable_threads=single -fi - -# Runtime is an alias for multiple with an additional value -if test "$enable_threads" = "runtime" ; then - AC_DEFINE(HAVE_RUNTIME_THREADCHECK,1,[Define if MPI supports MPI_THREAD_MULTIPLE with a runtime check for thread level]) - enable_threads=multiple -fi - MPICH_THREAD_LEVEL=MPI_THREAD_FUNNELED case "$enable_threads" in single) diff --git a/src/include/mpiimplthread.h b/src/include/mpiimplthread.h index 1f81e33..f0141c6 100644 --- a/src/include/mpiimplthread.h +++ b/src/include/mpiimplthread.h @@ -72,10 +72,11 @@ typedef struct MPICH_ThreadInfo_t { # endif /* !TLS */ MPID_Thread_id_t master_thread; /* Thread that started MPI */ #endif -#ifdef HAVE_RUNTIME_THREADCHECK + +#if defined MPICH_IS_THREADED int isThreaded; /* Set to true if user requested THREAD_MULTIPLE */ -#endif +#endif /* MPICH_IS_THREADED */ /* Define the mutex values used for each kind of implementation */ #if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL || \ @@ -107,11 +108,6 @@ extern MPICH_ThreadInfo_t MPIR_ThreadInfo; #else /* defined(MPICH_IS_THREADED) */ -/* We want to avoid the overhead of the thread call if we're in the - runtime state and threads are not in use. In that case, MPIR_Thread - is still a pointer but it was already allocated in InitThread */ -#ifdef HAVE_RUNTIME_THREADCHECK - #define MPIU_THREAD_CHECK_BEGIN if (MPIR_ThreadInfo.isThreaded) { #define MPIU_THREAD_CHECK_END } /* This macro used to take an argument of one or more statements that @@ -124,14 +120,6 @@ extern MPICH_ThreadInfo_t MPIR_ThreadInfo; * completely eliminate unnecessary if's. */ #define MPIU_ISTHREADED (MPIR_ThreadInfo.isThreaded) -#else /* !defined(HAVE_RUNTIME_THREADCHECK) */ - -#define MPIU_THREAD_CHECK_BEGIN -#define MPIU_THREAD_CHECK_END -#define MPIU_ISTHREADED (1) - -#endif /* HAVE_RUNTIME_THREADCHECK */ - #endif /* MPICH_IS_THREADED */ /* ------------------------------------------------------------------------- */ @@ -218,7 +206,6 @@ extern MPICH_PerThread_t MPIR_Thread; void MPIR_CleanupThreadStorage(void *a); #if !defined(MPIU_TLS_SPECIFIER) -# if defined(HAVE_RUNTIME_THREADCHECK) /* In the case where the thread level is set in MPI_Init_thread, we need a blended version of the non-threaded and the thread-multiple definitions. @@ -268,36 +255,6 @@ extern MPICH_PerThread_t MPIR_ThreadSingle; } \ } while (0) -# else /* unconditional thread multiple */ - -/* We initialize the MPIR_Thread pointer to null so that we need call the - * routine to get the thread-private storage only once in an invocation of a - * routine. */ -#define MPIU_THREADPRIV_INITKEY \ - do { \ - int err_; \ - MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_ThreadInfo.thread_storage,&err_); \ - MPIU_Assert(err_ == 0); \ - } while (0) -#define MPIU_THREADPRIV_INIT \ - do { \ - MPIR_Thread = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t)); \ - MPIU_Assert(MPIR_Thread); \ - MPID_Thread_tls_set(&MPIR_ThreadInfo.thread_storage, (void *)MPIR_Thread); \ - } while (0) -#define MPIU_THREADPRIV_GET \ - do { \ - if (!MPIR_Thread) { \ - MPID_Thread_tls_get(&MPIR_ThreadInfo.thread_storage, &MPIR_Thread); \ - if (!MPIR_Thread) { \ - MPIU_THREADPRIV_INIT; /* subtle, sets MPIR_Thread */ \ - } \ - MPIU_Assert(MPIR_Thread); \ - } \ - } while (0) - -# endif /* runtime vs. unconditional */ - /* common definitions when using MPID_Thread-based TLS */ #define MPIU_THREADPRIV_DECL MPICH_PerThread_t *MPIR_Thread=NULL #define MPIU_THREADPRIV_FIELD(a_) (MPIR_Thread->a_) diff --git a/src/include/mpitimpl.h b/src/include/mpitimpl.h index d7c3844..8de41f8 100644 --- a/src/include/mpitimpl.h +++ b/src/include/mpitimpl.h @@ -1363,14 +1363,9 @@ static inline int MPIR_T_is_initialized() { extern void MPIR_T_strncpy(char *dst, const char *src, int *len); /* Stuffs to support multithreaded MPI_T */ -#ifdef HAVE_RUNTIME_THREADCHECK extern int MPIR_T_is_threaded; #define MPIR_T_THREAD_CHECK_BEGIN if (MPIR_T_is_threaded) { #define MPIR_T_THREAD_CHECK_END } -#else /* !HAVE_RUNTIME_THREADCHECK */ -#define MPIR_T_THREAD_CHECK_BEGIN -#define MPIR_T_THREAD_CHECK_END -#endif #ifdef MPICH_IS_THREADED extern MPIU_Thread_mutex_t mpi_t_mutex; diff --git a/src/mpi/init/init.c b/src/mpi/init/init.c index f9d1c9a..a5af3da 100644 --- a/src/mpi/init/init.c +++ b/src/mpi/init/init.c @@ -146,12 +146,9 @@ int MPI_Init( int *argc, char ***argv ) * mutexes are not initialized yet, and we don't want to * accidentally use them before they are initialized. We will * reset this value once it is properly initialized. */ - /* FIXME: This only works when runtime thread-safety is enabled. - * When we use configure-time thread-levels, we might still have a - * problem. */ -#ifdef HAVE_RUNTIME_THREADCHECK +#if defined MPICH_IS_THREADED MPIR_ThreadInfo.isThreaded = 0; -#endif +#endif /* MPICH_IS_THREADED */ MPIR_T_env_init(); diff --git a/src/mpi/init/initthread.c b/src/mpi/init/initthread.c index 98ca462..231e214 100644 --- a/src/mpi/init/initthread.c +++ b/src/mpi/init/initthread.c @@ -160,7 +160,7 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved) MPICH_PerThread_t MPIR_Thread = { 0 }; #elif defined(MPIU_TLS_SPECIFIER) MPIU_TLS_SPECIFIER MPICH_PerThread_t MPIR_Thread = { 0 }; -#elif defined(HAVE_RUNTIME_THREADCHECK) +#else /* If we may be single threaded, we need a preallocated version to use if we are single threaded case */ MPICH_PerThread_t MPIR_ThreadSingle = { 0 }; @@ -302,9 +302,9 @@ int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided) /* For any code in the device that wants to check for runtime decisions on the value of isThreaded, set a provisional value here. We could let the MPID_Init routine override this */ -#ifdef HAVE_RUNTIME_THREADCHECK +#if defined MPICH_IS_THREADED MPIR_ThreadInfo.isThreaded = required == MPI_THREAD_MULTIPLE; -#endif +#endif /* MPICH_IS_THREADED */ MPIU_THREAD_CS_INIT; @@ -479,9 +479,9 @@ int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided) /* Capture the level of thread support provided */ MPIR_ThreadInfo.thread_provided = thread_provided; if (provided) *provided = thread_provided; -#ifdef HAVE_RUNTIME_THREADCHECK +#if defined MPICH_IS_THREADED MPIR_ThreadInfo.isThreaded = (thread_provided == MPI_THREAD_MULTIPLE); -#endif +#endif /* MPICH_IS_THREADED */ /* FIXME: Define these in the interface. Does Timer init belong here? */ MPIU_dbg_init(MPIR_Process.comm_world->rank); @@ -618,12 +618,9 @@ int MPI_Init_thread( int *argc, char ***argv, int required, int *provided ) * mutexes are not initialized yet, and we don't want to * accidentally use them before they are initialized. We will * reset this value once it is properly initialized. */ - /* FIXME: This only works when runtime thread-safety is enabled. - * When we use configure-time thread-levels, we might still have a - * problem. */ -#ifdef HAVE_RUNTIME_THREADCHECK +#if defined MPICH_IS_THREADED MPIR_ThreadInfo.isThreaded = 0; -#endif +#endif /* MPICH_IS_THREADED */ MPIR_T_env_init(); diff --git a/src/mpi_t/mpit.c b/src/mpi_t/mpit.c index 2d580de..62f602c 100644 --- a/src/mpi_t/mpit.c +++ b/src/mpi_t/mpit.c @@ -10,9 +10,6 @@ int MPIR_T_init_balance = 0; #ifdef MPICH_IS_THREADED MPIU_Thread_mutex_t mpi_t_mutex; -#endif - -#ifdef HAVE_RUNTIME_THREADCHECK int MPIR_T_is_threaded; #endif diff --git a/src/mpi_t/mpit_initthread.c b/src/mpi_t/mpit_initthread.c index 23ab02e..1e7fcba 100644 --- a/src/mpi_t/mpit_initthread.c +++ b/src/mpi_t/mpit_initthread.c @@ -118,9 +118,9 @@ int MPI_T_init_thread(int required, int *provided) /* ... body of routine ... */ -#ifdef HAVE_RUNTIME_THREADCHECK +#if defined MPICH_IS_THREADED MPIR_T_is_threaded = (required == MPI_THREAD_MULTIPLE); -#endif +#endif /* MPICH_IS_THREADED */ if (provided != NULL) { /* This must be min(required,MPICH_THREAD_LEVEL) if runtime diff --git a/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h b/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h index f702e1f..7f21c6d 100644 --- a/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h +++ b/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h @@ -878,11 +878,7 @@ MPID_nem_mpich_blocking_recv(MPID_nem_cell_ptr_t *cell, int *in_fbox, int comple #ifdef MPICH_IS_THREADED /* We should never enter this function in a multithreaded app */ -#ifdef HAVE_RUNTIME_THREADCHECK MPIU_Assert(!MPIR_ThreadInfo.isThreaded); -#else - MPIU_Assert(0); -#endif #endif #ifdef USE_FASTBOX diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c index 7e9cf28..785476a 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c @@ -350,11 +350,7 @@ int MPIDI_CH3I_Progress (MPID_Progress_state *progress_state, int is_blocking) /* check queue */ if (MPID_nem_safe_to_block_recv() && is_blocking #ifdef MPICH_IS_THREADED -#ifdef HAVE_RUNTIME_THREADCHECK && !MPIR_ThreadInfo.isThreaded -#else - && 0 -#endif #endif ) { diff --git a/src/mpid/ch3/channels/sock/src/ch3_progress.c b/src/mpid/ch3/channels/sock/src/ch3_progress.c index 0311856..fec55f7 100644 --- a/src/mpid/ch3/channels/sock/src/ch3_progress.c +++ b/src/mpid/ch3/channels/sock/src/ch3_progress.c @@ -188,7 +188,6 @@ static int MPIDI_CH3i_Progress_wait(MPID_Progress_state * progress_state) /* The logic for this case is just complicated enough that we write separate code for each possibility */ -# ifdef HAVE_RUNTIME_THREADCHECK if (MPIR_ThreadInfo.isThreaded) { MPIDI_CH3I_progress_blocked = TRUE; mpi_errno = MPIDU_Sock_wait(MPIDI_CH3I_sock_set, @@ -200,13 +199,6 @@ static int MPIDI_CH3i_Progress_wait(MPID_Progress_state * progress_state) mpi_errno = MPIDU_Sock_wait(MPIDI_CH3I_sock_set, MPIDU_SOCK_INFINITE_TIME, &event); } -# else - MPIDI_CH3I_progress_blocked = TRUE; - mpi_errno = MPIDU_Sock_wait(MPIDI_CH3I_sock_set, - MPIDU_SOCK_INFINITE_TIME, &event); - MPIDI_CH3I_progress_blocked = FALSE; - MPIDI_CH3I_progress_wakeup_signalled = FALSE; -# endif /* HAVE_RUNTIME_THREADCHECK */ # else mpi_errno = MPIDU_Sock_wait(MPIDI_CH3I_sock_set, diff --git a/src/mpid/common/sock/poll/sock_wait.i b/src/mpid/common/sock/poll/sock_wait.i index 393e71a..7a7e85d 100644 --- a/src/mpid/common/sock/poll/sock_wait.i +++ b/src/mpid/common/sock/poll/sock_wait.i @@ -118,7 +118,6 @@ int MPIDU_Sock_wait(struct MPIDU_Sock_set * sock_set, int millisecond_timeout, just use the same code as above. Otherwise, use multithreaded code (and we don't then need the MPIU_THREAD_CHECK_BEGIN/END macros) */ -#ifdef HAVE_RUNTIME_THREADCHECK if (!MPIR_ThreadInfo.isThreaded) { MPIDI_FUNC_ENTER(MPID_STATE_POLL); n_fds = poll(sock_set->pollfds, sock_set->poll_array_elems, @@ -126,7 +125,6 @@ int MPIDU_Sock_wait(struct MPIDU_Sock_set * sock_set, int millisecond_timeout, MPIDI_FUNC_EXIT(MPID_STATE_POLL); } else -#endif { /* * First try a non-blocking poll to see if any immediate diff --git a/src/mpid/pamid/include/mpidi_thread.h b/src/mpid/pamid/include/mpidi_thread.h index aaa1c2f..eb31b24 100644 --- a/src/mpid/pamid/include/mpidi_thread.h +++ b/src/mpid/pamid/include/mpidi_thread.h @@ -46,9 +46,6 @@ #if (MPICH_THREAD_LEVEL != MPI_THREAD_MULTIPLE) #error MPICH_THREAD_LEVEL should be MPI_THREAD_MULTIPLE #endif -#ifndef HAVE_RUNTIME_THREADCHECK -#error Need HAVE_RUNTIME_THREADCHECK -#endif #define MPIU_THREAD_CS_INIT ({ MPIDI_Mutex_initialize(); }) #define MPIU_THREAD_CS_FINALIZE http://git.mpich.org/mpich.git/commitdiff/99485af86e826a946fe038a77f0b98244… commit 99485af86e826a946fe038a77f0b98244526dc4d Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Tue Jan 7 04:58:42 2014 +0000 Thread critical-section initialization fixes. This patch has two related parts. 1. We initialize the isThreaded runtime check to FALSE before doing the cvar checks. This is because the cvar initialization uses memory allocation and string duping, which internally use mutexes. But the mutexes are not initialized yet, so disabling isThreaded would make sure we don't use them. 2. Updates to the pamid device to respect the isThreaded variable. This is needed since we were initializing cvars before setting the thread-level, so the thread-safety macros are not initialized at that point. See #1900. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/init/init.c b/src/mpi/init/init.c index 56a87d0..f9d1c9a 100644 --- a/src/mpi/init/init.c +++ b/src/mpi/init/init.c @@ -142,6 +142,17 @@ int MPI_Init( int *argc, char ***argv ) /* ... body of routine ... */ + /* Temporarily disable thread-safety. This is needed because the + * mutexes are not initialized yet, and we don't want to + * accidentally use them before they are initialized. We will + * reset this value once it is properly initialized. */ + /* FIXME: This only works when runtime thread-safety is enabled. + * When we use configure-time thread-levels, we might still have a + * problem. */ +#ifdef HAVE_RUNTIME_THREADCHECK + MPIR_ThreadInfo.isThreaded = 0; +#endif + MPIR_T_env_init(); if (!strcmp(MPIR_CVAR_DEFAULT_THREAD_LEVEL, "MPI_THREAD_MULTIPLE")) diff --git a/src/mpi/init/initthread.c b/src/mpi/init/initthread.c index 3ebe996..98ca462 100644 --- a/src/mpi/init/initthread.c +++ b/src/mpi/init/initthread.c @@ -614,6 +614,17 @@ int MPI_Init_thread( int *argc, char ***argv, int required, int *provided ) /* ... body of routine ... */ + /* Temporarily disable thread-safety. This is needed because the + * mutexes are not initialized yet, and we don't want to + * accidentally use them before they are initialized. We will + * reset this value once it is properly initialized. */ + /* FIXME: This only works when runtime thread-safety is enabled. + * When we use configure-time thread-levels, we might still have a + * problem. */ +#ifdef HAVE_RUNTIME_THREADCHECK + MPIR_ThreadInfo.isThreaded = 0; +#endif + MPIR_T_env_init(); /* If the user requested for asynchronous progress, request for diff --git a/src/mpid/pamid/include/mpidi_thread.h b/src/mpid/pamid/include/mpidi_thread.h index 5f3fb7e..aaa1c2f 100644 --- a/src/mpid/pamid/include/mpidi_thread.h +++ b/src/mpid/pamid/include/mpidi_thread.h @@ -50,7 +50,6 @@ #error Need HAVE_RUNTIME_THREADCHECK #endif - #define MPIU_THREAD_CS_INIT ({ MPIDI_Mutex_initialize(); }) #define MPIU_THREAD_CS_FINALIZE @@ -111,19 +110,55 @@ #elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT -#define MPIDI_CS_ENTER(m) ({ MPIDI_Mutex_acquire(m); }) -#define MPIDI_CS_EXIT(m) ({ MPIDI_Mutex_sync(); MPIDI_Mutex_release(m); }) -#define MPIDI_CS_YIELD(m) ({ MPIDI_Mutex_sync(); MPIDI_Mutex_release(m); MPIDI_Mutex_acquire(m); }) -#define MPIDI_CS_TRY(m) ({ (0==MPIDI_Mutex_try_acquire(m)); }) -#define MPIDI_CS_SCHED_YIELD(m) ({ MPIDI_Mutex_sync(); MPIDI_Mutex_release(m); sched_yield(); MPIDI_Mutex_acquire(m); }) +#define MPIDI_CS_ENTER(m) \ + do { \ + MPIU_THREAD_CHECK_BEGIN \ + MPIDI_Mutex_acquire(m); \ + MPIU_THREAD_CHECK_END \ + } while (0) + +#define MPIDI_CS_EXIT(m) \ + do { \ + MPIU_THREAD_CHECK_BEGIN \ + MPIDI_Mutex_sync(); \ + MPIDI_Mutex_release(m); \ + MPIU_THREAD_CHECK_END \ + } while (0) + +#define MPIDI_CS_YIELD(m) \ + do { \ + MPIU_THREAD_CHECK_BEGIN \ + MPIDI_Mutex_sync(); \ + MPIDI_Mutex_release(m); \ + MPIDI_Mutex_acquire(m); \ + MPIU_THREAD_CHECK_END \ + } while (0) + +#define MPIDI_CS_TRY(m) \ + do { \ + MPIU_THREAD_CHECK_BEGIN \ + MPIDI_Mutex_try_acquire(m); \ + MPIU_THREAD_CHECK_END \ + } while (0) + +#define MPIDI_CS_SCHED_YIELD(m) \ + do { \ + MPIU_THREAD_CHECK_BEGIN \ + MPIDI_Mutex_sync(); \ + MPIDI_Mutex_release(m); \ + sched_yield(); \ + MPIDI_Mutex_acquire(m); \ + MPIU_THREAD_CHECK_END \ + } while (0) #define MPIU_THREAD_CS_ALLFUNC_ENTER(_context) #define MPIU_THREAD_CS_ALLFUNC_EXIT(_context) #define MPIU_THREAD_CS_ALLFUNC_YIELD(_context) #define MPIU_THREAD_CS_ALLFUNC_SCHED_YIELD(_context) #define MPIU_THREAD_CS_ALLFUNC_TRY(_context) (0) -#define MPIU_THREAD_CS_INIT_ENTER(_context) MPIDI_Mutex_acquire(0) -#define MPIU_THREAD_CS_INIT_EXIT(_context) MPIDI_Mutex_release(0) + +#define MPIU_THREAD_CS_INIT_ENTER(_context) MPIDI_CS_ENTER(0) +#define MPIU_THREAD_CS_INIT_EXIT(_context) MPIDI_CS_EXIT(0) #define MPIU_THREAD_CS_CONTEXTID_ENTER(_context) MPIDI_CS_ENTER(0) #define MPIU_THREAD_CS_CONTEXTID_EXIT(_context) MPIDI_CS_EXIT (0) ----------------------------------------------------------------------- Summary of changes: configure.ac | 30 ++--------- src/include/mpiimplthread.h | 49 +----------------- src/include/mpitimpl.h | 5 -- src/mpi/init/init.c | 8 +++ src/mpi/init/initthread.c | 18 +++++-- src/mpi_t/mpit.c | 3 - src/mpi_t/mpit_initthread.c | 4 +- .../ch3/channels/nemesis/include/mpid_nem_inline.h | 4 -- src/mpid/ch3/channels/nemesis/src/ch3_progress.c | 4 -- src/mpid/ch3/channels/sock/src/ch3_progress.c | 8 --- src/mpid/common/sock/poll/sock_wait.i | 2 - src/mpid/pamid/include/mpidi_thread.h | 54 ++++++++++++++++---- 12 files changed, 75 insertions(+), 114 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-150-g3b24d31
by noreply＠mpich.org 10 Jan '14

10 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3b24d31851794830cb449b03ba09bfc6cf1168f0 (commit) from cb15d17c08c2f1a52ddc80cdc2eddff472c3b3c1 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3b24d31851794830cb449b03ba09bfc6c… commit 3b24d31851794830cb449b03ba09bfc6cf1168f0 Author: Wesley Bland <wbland(a)mcs.anl.gov> Date: Fri Jan 10 14:55:40 2014 -0600 Mark ft/sendalive as xfail for now. This test still needs to be corrected, but for now it will be xfail. It is being tracked with ticket #1996. No reviewer diff --git a/test/mpi/ft/testlist b/test/mpi/ft/testlist index 4b662d0..94294ed 100644 --- a/test/mpi/ft/testlist +++ b/test/mpi/ft/testlist @@ -1,6 +1,6 @@ die 4 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false resultTest=TestStatusNoErrors xfail=ticket1996 abort 2 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false xfail=ticket1537 -sendalive 4 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false resultTest=TestStatusNoErrors +sendalive 4 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false resultTest=TestStatusNoErrors xfail=ticket=1996 isendalive 3 mpiexecarg=-disable-auto-cleanup resultTest=TestStatusNoErrors strict=false senddead 2 mpiexecarg=-disable-auto-cleanup resultTest=TestStatusNoErrors strict=false timeLimit=10 xfail=ticket1945 recvdead 2 mpiexecarg=-disable-auto-cleanup resultTest=TestStatusNoErrors strict=false timeLimit=10 ----------------------------------------------------------------------- Summary of changes: test/mpi/ft/testlist | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-149-gcb15d17
by noreply＠mpich.org 09 Jan '14

09 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via cb15d17c08c2f1a52ddc80cdc2eddff472c3b3c1 (commit) from 28c721724cacd8c07f27a5aee536829c987eba09 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/cb15d17c08c2f1a52ddc80cdc2eddff47… commit cb15d17c08c2f1a52ddc80cdc2eddff472c3b3c1 Author: Wesley Bland <wbland(a)mcs.anl.gov> Date: Thu Jan 9 14:53:42 2014 -0600 Mark the ft/die test as xfail for now. This test needs to be debugged, but it can wait for v3.1.1 since this is an experimental failure. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/test/mpi/ft/testlist b/test/mpi/ft/testlist index 8c00b02..4b662d0 100644 --- a/test/mpi/ft/testlist +++ b/test/mpi/ft/testlist @@ -1,4 +1,4 @@ -die 4 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false resultTest=TestStatusNoErrors +die 4 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false resultTest=TestStatusNoErrors xfail=ticket1996 abort 2 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false xfail=ticket1537 sendalive 4 mpiexecarg=-disable-auto-cleanup timeLimit=10 strict=false resultTest=TestStatusNoErrors isendalive 3 mpiexecarg=-disable-auto-cleanup resultTest=TestStatusNoErrors strict=false ----------------------------------------------------------------------- Summary of changes: test/mpi/ft/testlist | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-148-g28c7217
by noreply＠mpich.org 09 Jan '14

09 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 28c721724cacd8c07f27a5aee536829c987eba09 (commit) from 4b07ac8ab4ba5203f228348b05a9a2da71f13396 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/28c721724cacd8c07f27a5aee536829c9… commit 28c721724cacd8c07f27a5aee536829c987eba09 Author: Su Huang <suhuang(a)us.ibm.com> Date: Thu Jan 9 11:59:11 2014 -0500 PAMID:fix memory leak and BGQ problems Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c index 7792d8f..2d04091 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c @@ -193,6 +193,13 @@ MPID_Accumulate(void *origin_addr, } req->offset = target_disp * win->mpid.info[target_rank].disp_unit; +#ifdef __BGQ__ + /* PAMI limitation as it doesnt permit VA of 0 to be passed into + * memregion create, so we must pass base_va of heap computed from + * an SPI call instead. So the target offset must be adjusted */ + if (req->win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) + req->offset -= (size_t)req->win->mpid.info[target_rank].base_addr; +#endif if (origin_datatype == MPI_DOUBLE_INT) { diff --git a/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c b/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c index 3f7d925..4c52993 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c +++ b/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c @@ -95,6 +95,13 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, req->type = MPIDI_WIN_REQUEST_COMPARE_AND_SWAP; req->offset = target_disp * win->mpid.info[target_rank].disp_unit; +#ifdef __BGQ__ + /* PAMI limitation as it doesnt permit VA of 0 to be passed into + * memregion create, so we must pass base_va of heap computed from + * an SPI call instead. So the target offset must be adjusted */ + if (req->win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) + req->offset -= (size_t)req->win->mpid.info[target_rank].base_addr; +#endif MPIDI_Win_datatype_basic(1, datatype, &req->origin.dt); @@ -119,6 +126,10 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, base = win->mpid.info[target_rank].base_addr; disp_unit = win->disp_unit; } + else if (win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) { + base = NULL; + disp_unit = win->disp_unit; + } else { base = win->base; disp_unit = win->disp_unit; @@ -178,10 +189,13 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPI_Op null_op=0; pami_data_function pami_op; +#ifndef __BGQ__ if(MPIDI_Datatype_is_pami_rmw_supported(basic_type, &pami_type, null_op, &pami_op) ) { req->pami_datatype = pami_type; PAMI_Context_post(MPIDI_Context[0], &req->post_request, MPIDI_Compare_and_swap_using_pami_rmw, req); - } else { + } else +#endif + { PAMI_Context_post(MPIDI_Context[0], &req->post_request, MPIDI_Atomic, req); } } diff --git a/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c b/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c index 74e10ca..4b37f26 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c +++ b/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c @@ -52,11 +52,34 @@ MPID_Win_create_dynamic( MPID_Info * info, rc=MPIDI_Win_init(0,1,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_DYNAMIC, MPI_WIN_UNIFIED); win = *win_ptr; - win->base = MPI_BOTTOM; rank = comm_ptr->rank; + win->base = MPI_BOTTOM; winfo = &win->mpid.info[rank]; winfo->win = win; +#ifdef __BGQ__ + void *base = NULL; + size_t length = 0; + Kernel_MemoryRegion_t memregion; + void *tmpbuf = malloc(sizeof(int)); + Kernel_CreateMemoryRegion(&memregion, tmpbuf, sizeof(int)); + //Reset base to base VA of local heap + base = memregion.BaseVa; + length = memregion.Bytes; + free(tmpbuf); + + if (length != 0) + { + size_t length_out = 0; + pami_result_t rc; + rc = PAMI_Memregion_create(MPIDI_Context[0], base, length, &length_out, &winfo->memregion); + MPID_assert(rc == PAMI_SUCCESS); + MPID_assert(length == length_out); + } + win->base = base; + winfo->base_addr = base; +#endif + rc= MPIDI_Win_allgather(0,win_ptr); if (rc != MPI_SUCCESS) return rc; diff --git a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c index 9f47b9b..a99a19e 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c +++ b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c @@ -229,7 +229,9 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, MPID_assert(basic_type != MPI_DATATYPE_NULL); if(MPIDI_Datatype_is_pami_rmw_supported(basic_type, &pami_type, op, &pami_op) ) { +#ifndef __BGQ__ good_for_rmw = 1; +#endif } else { if((op == MPI_NO_OP) && (origin_addr == NULL) && (win->create_flavor != MPI_WIN_FLAVOR_SHARED) ) { /* essentially a MPI_Get to result buffer */ @@ -244,6 +246,13 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, req->type = MPIDI_WIN_REQUEST_FETCH_AND_OP; req->offset = target_disp * win->mpid.info[target_rank].disp_unit; +#ifdef __BGQ__ + /* PAMI limitation as it doesnt permit VA of 0 to be passed into + * memregion create, so we must pass base_va of heap computed from + * an SPI call instead. So the target offset must be adjusted */ + if (req->win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) + req->offset -= (size_t)req->win->mpid.info[target_rank].base_addr; +#endif if (datatype == MPI_DOUBLE_INT) { @@ -300,6 +309,10 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, disp_unit = win->disp_unit; } + else if (win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) { + base = NULL; + disp_unit = win->disp_unit; + } else { base = win->base; disp_unit = win->disp_unit; diff --git a/src/mpid/pamid/src/onesided/mpid_win_get.c b/src/mpid/pamid/src/onesided/mpid_win_get.c index b56e2b2..ab341f7 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get.c @@ -236,6 +236,13 @@ MPID_Get(void *origin_addr, } req->offset = target_disp * win->mpid.info[target_rank].disp_unit; +#ifdef __BGQ__ + /* PAMI limitation as it doesnt permit VA of 0 to be passed into + * memregion create, so we must pass base_va of heap computed from + * an SPI call instead. So the target offset must be adjusted */ + if (req->win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) + req->offset -= (size_t)req->win->mpid.info[target_rank].base_addr; +#endif MPIDI_Win_datatype_basic(origin_count, origin_datatype, diff --git a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c index 8f8d07c..ca93a13 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c @@ -23,10 +23,12 @@ static void MPIDI_Win_GetAccSendAckDoneCB(pami_context_t context, - void * _buf, + void * _info, pami_result_t result) { - MPIU_Free (_buf); + MPIDI_Win_GetAccMsgInfo *msginfo = (MPIDI_Win_GetAccMsgInfo *) _info; + MPIU_Free(msginfo->tptr); + MPIU_Free(msginfo); } static void @@ -69,19 +71,18 @@ MPIDI_Win_GetAccumSendAck(pami_context_t context, .dest = msginfo->src_endpoint, }, .events = { - .cookie = buffer, + .cookie = msginfo, .local_fn = MPIDI_Win_GetAccSendAckDoneCB, //cleanup buffer }, }; + msginfo->tptr = buffer; + params.send.data.iov_len = msginfo->size; params.send.data.iov_base = buffer; rc = PAMI_Send(context, &params); MPID_assert(rc == PAMI_SUCCESS); - - //free msginfo - //MPIU_Free(msginfo); } void @@ -148,6 +149,9 @@ MPIDI_Win_GetAccDoneCB(pami_context_t context, if (req->accum_headers) MPIU_Free(req->accum_headers); + MPIDI_Win_datatype_unmap(&req->target.dt); + MPIDI_Win_datatype_unmap(&req->result.dt); + if( req->type != MPIDI_WIN_REQUEST_RGET_ACCUMULATE ) MPIU_Free(req); } @@ -166,10 +170,10 @@ MPIDI_Win_GetAccAckDoneCB(pami_context_t context, MPIR_Localcopy(req->result.addr, req->result.count, req->result.datatype, - msginfo->result_addr, + msginfo->tptr, msginfo->size, MPI_CHAR); - MPIU_Free(msginfo->result_addr); + MPIU_Free(msginfo->tptr); } MPIU_Free(msginfo); @@ -194,10 +198,10 @@ MPIDI_WinGetAccumAckCB(pami_context_t context, *msginfo = *(const MPIDI_Win_GetAccMsgInfo *)_msginfo; MPIDI_Win_request *req = (MPIDI_Win_request *) msginfo->request; - msginfo->result_addr = NULL; + msginfo->tptr = NULL; recv->addr = req->result.addr; if (req->result_num_contig > 1) - recv->addr = msginfo->result_addr = MPIU_Malloc(msginfo->size); + recv->addr = msginfo->tptr = MPIU_Malloc(msginfo->size); recv->type = PAMI_TYPE_BYTE; recv->offset = 0; @@ -213,7 +217,6 @@ MPIDI_Get_accumulate(pami_context_t context, { MPIDI_Win_request *req = (MPIDI_Win_request*)_req; pami_result_t rc; - void *map; pami_send_t params = { .send = { @@ -248,15 +251,8 @@ MPIDI_Get_accumulate(pami_context_t context, params.send.data.iov_base = req->buffer + req->state.local_offset; if (req->target.dt.num_contig - req->state.index == 1) { - map=NULL; - if (req->target.dt.map != &req->target.dt.__map) { - map=(void *) req->target.dt.map; - } - rc = PAMI_Send(context, &params); MPID_assert(rc == PAMI_SUCCESS); - if (map) - MPIU_Free(map); return PAMI_SUCCESS; } else { rc = PAMI_Send(context, &params); @@ -265,9 +261,6 @@ MPIDI_Get_accumulate(pami_context_t context, ++req->state.index; } } - - MPIDI_Win_datatype_unmap(&req->target.dt); - return PAMI_SUCCESS; } diff --git a/src/mpid/pamid/src/onesided/mpid_win_put.c b/src/mpid/pamid/src/onesided/mpid_win_put.c index 67f9d84..110f294 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_put.c +++ b/src/mpid/pamid/src/onesided/mpid_win_put.c @@ -249,6 +249,13 @@ MPID_Put(void *origin_addr, } req->offset = target_disp * win->mpid.info[target_rank].disp_unit; +#ifdef __BGQ__ + /* PAMI limitation as it doesnt permit VA of 0 to be passed into + * memregion create, so we must pass base_va of heap computed from + * an SPI call instead. So the target offset must be adjusted */ + if (req->win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) + req->offset -= (size_t)req->win->mpid.info[target_rank].base_addr; +#endif MPIDI_Win_datatype_basic(origin_count, origin_datatype, diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 180aaad..8604bb5 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -201,7 +201,7 @@ typedef struct int num_contig; int size; void * request; - void * result_addr; + void * tptr; pami_endpoint_t src_endpoint; } MPIDI_Win_GetAccMsgInfo; ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/src/onesided/mpid_win_accumulate.c | 7 ++++ .../pamid/src/onesided/mpid_win_compare_and_swap.c | 16 ++++++++- .../pamid/src/onesided/mpid_win_create_dynamic.c | 25 +++++++++++++- .../pamid/src/onesided/mpid_win_fetch_and_op.c | 13 +++++++ src/mpid/pamid/src/onesided/mpid_win_get.c | 7 ++++ .../pamid/src/onesided/mpid_win_get_accumulate.c | 35 ++++++++------------ src/mpid/pamid/src/onesided/mpid_win_put.c | 7 ++++ src/mpid/pamid/src/onesided/mpidi_onesided.h | 2 +- 8 files changed, 88 insertions(+), 24 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-147-g4b07ac8
by noreply＠mpich.org 08 Jan '14

08 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 4b07ac8ab4ba5203f228348b05a9a2da71f13396 (commit) via c03e766f497f91de0f75a37c945a8e356f1d3788 (commit) via 039c2290692c6fff053fd62ec2a3b5da700c7163 (commit) via e25dc31f0ba40a35086b60e76afe1069af876b58 (commit) via 31ac6f01a51883c466255cd8ae80ae32a9d7a6a9 (commit) via c6d910c783380440d1946ab366e5f2496eaed042 (commit) from 399e546369c3938ada9cc0b6c3c218f685c7613b (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/4b07ac8ab4ba5203f228348b05a9a2da7… commit 4b07ac8ab4ba5203f228348b05a9a2da71f13396 Author: Sameer Kumar <sameerk(a)us.ibm.com> Date: Tue Jan 7 04:29:13 2014 -0600 Bug fix for strided datatypes. Full fix Fix for get accumulate that sends contig ack back and then scatters result buffer on the src node. Remove unused params. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_1s.c b/src/mpid/pamid/src/onesided/mpid_1s.c index 3b89aab..6e64c3f 100644 --- a/src/mpid/pamid/src/onesided/mpid_1s.c +++ b/src/mpid/pamid/src/onesided/mpid_1s.c @@ -45,7 +45,9 @@ MPIDI_Win_DoneCB(pami_context_t context, req->origin.count, req->origin.datatype); MPID_assert(mpi_errno == MPI_SUCCESS); +#ifndef USE_PAMI_RDMA MPIDI_Win_datatype_unmap(&req->target.dt); +#endif MPID_Datatype_release(req->origin.dt.pointer); MPIU_Free(req->buffer); MPIU_Free(req->user_buffer); diff --git a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c index b8779e1..8f8d07c 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c @@ -38,28 +38,21 @@ MPIDI_Win_GetAccumSendAck(pami_context_t context, pami_result_t rc = PAMI_SUCCESS; //Copy from msginfo->addr to a contiguous buffer - MPIDI_Datatype result_dt; char *buffer = NULL; - MPIDI_Win_datatype_basic(msginfo->result_count, - msginfo->result_datatype, - &result_dt); - - int use_map = 0; - buffer = MPIU_Malloc(result_dt.size); - if (result_dt.contig) - memcpy(buffer, (msginfo->addr + result_dt.true_lb), result_dt.size); + buffer = MPIU_Malloc(msginfo->size); + MPID_assert(buffer != NULL); + + if (msginfo->num_contig == 1) + memcpy(buffer, msginfo->addr, msginfo->size); else { - use_map = 1; - MPID_assert(buffer != NULL); - int mpi_errno = 0; mpi_errno = MPIR_Localcopy(msginfo->addr, msginfo->count, msginfo->type, buffer, - result_dt.size, + msginfo->size, MPI_CHAR); MPID_assert(mpi_errno == MPI_SUCCESS); } @@ -69,6 +62,7 @@ MPIDI_Win_GetAccumSendAck(pami_context_t context, pami_send_t params = { .send = { .header = { + .iov_base = msginfo, .iov_len = sizeof(MPIDI_Win_GetAccMsgInfo), }, .dispatch = MPIDI_Protocols_WinGetAccumAck, @@ -80,30 +74,14 @@ MPIDI_Win_GetAccumSendAck(pami_context_t context, }, }; - int index = 0; - size_t local_offset = 0; - //Set the map - MPIDI_Win_datatype_map(&result_dt); - MPID_assert(result_dt.num_contig == msginfo->result_num_contig); - - while (index < result_dt.num_contig) { - params.send.header.iov_base = msginfo; - params.send.data.iov_len = result_dt.map[index].DLOOP_VECTOR_LEN; - params.send.data.iov_base = buffer + local_offset; - - rc = PAMI_Send(context, &params); - MPID_assert(rc == PAMI_SUCCESS); - local_offset += params.send.data.iov_len; - ++index; - } - - /** Review PAMI_Send semantics and consider moving the free calls to - the completion callback*/ - //free msginfo - MPIU_Free(msginfo); + params.send.data.iov_len = msginfo->size; + params.send.data.iov_base = buffer; + + rc = PAMI_Send(context, &params); + MPID_assert(rc == PAMI_SUCCESS); - if (use_map && result_dt.map != &result_dt.__map) - MPIU_Free (result_dt.map); + //free msginfo + //MPIU_Free(msginfo); } void @@ -158,8 +136,7 @@ MPIDI_Win_GetAccDoneCB(pami_context_t context, ++req->win->mpid.sync.complete; ++req->origin.completed; - if (req->origin.completed == - (req->result_num_contig + req->target.dt.num_contig)) + if (req->origin.completed == req->target.dt.num_contig + 1) { if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); @@ -178,6 +155,29 @@ MPIDI_Win_GetAccDoneCB(pami_context_t context, } void +MPIDI_Win_GetAccAckDoneCB(pami_context_t context, + void * _msginfo, + pami_result_t result) +{ + MPIDI_Win_GetAccMsgInfo * msginfo =(MPIDI_Win_GetAccMsgInfo *)_msginfo; + MPIDI_Win_request *req = (MPIDI_Win_request *) msginfo->request; + + if (req->result_num_contig > 1) { + MPIR_Localcopy(req->result.addr, + req->result.count, + req->result.datatype, + msginfo->result_addr, + msginfo->size, + MPI_CHAR); + MPIU_Free(msginfo->result_addr); + } + MPIU_Free(msginfo); + + MPIDI_Win_GetAccDoneCB(context, req, result); +} + + +void MPIDI_WinGetAccumAckCB(pami_context_t context, void * cookie, const void * _msginfo, @@ -189,24 +189,22 @@ MPIDI_WinGetAccumAckCB(pami_context_t context, { MPID_assert(recv != NULL); MPID_assert(sndbuf == NULL); - MPID_assert(msginfo_size == sizeof(MPIDI_Win_GetAccMsgInfo)); MPID_assert(_msginfo != NULL); - const MPIDI_Win_GetAccMsgInfo * msginfo =(const MPIDI_Win_GetAccMsgInfo *)_msginfo; - - int null=0; - pami_type_t pami_type; - pami_data_function pami_op; - MPI_Op op = msginfo->op; - - MPIDI_Datatype_to_pami(msginfo->result_datatype, &pami_type, op, &pami_op, &null); - - recv->addr = msginfo->result_addr; - recv->type = pami_type; + MPIDI_Win_GetAccMsgInfo * msginfo =MPIU_Malloc(sizeof(MPIDI_Win_GetAccMsgInfo)); + *msginfo = *(const MPIDI_Win_GetAccMsgInfo *)_msginfo; + MPIDI_Win_request *req = (MPIDI_Win_request *) msginfo->request; + + msginfo->result_addr = NULL; + recv->addr = req->result.addr; + if (req->result_num_contig > 1) + recv->addr = msginfo->result_addr = MPIU_Malloc(msginfo->size); + + recv->type = PAMI_TYPE_BYTE; recv->offset = 0; recv->data_fn = PAMI_DATA_COPY; recv->data_cookie = NULL; - recv->local_fn = MPIDI_Win_GetAccDoneCB; - recv->cookie = msginfo->req; + recv->local_fn = MPIDI_Win_GetAccAckDoneCB; + recv->cookie = msginfo; } static pami_result_t @@ -455,13 +453,15 @@ MPID_Get_accumulate(const void * origin_addr, MPIDI_Win_datatype_map(&req->target.dt); - MPIDI_Datatype result_dt; - MPIDI_Win_datatype_basic(result_count, result_datatype, &result_dt); - req->result_num_contig = 1; - if (!result_dt.contig) - req->result_num_contig =result_dt.pointer->max_contig_blocks*result_count+1; + req->result.addr = result_addr; + req->result.count = result_count; + req->result.datatype = result_datatype; + MPIDI_Win_datatype_basic(result_count, result_datatype, &req->result.dt); + MPIDI_Win_datatype_map(&req->result.dt); + req->result_num_contig = req->result.dt.num_contig; + //We wait for #messages depending on target and result_datatype - win->mpid.sync.total += (req->result_num_contig + req->target.dt.num_contig); + win->mpid.sync.total += (1 + req->target.dt.num_contig); { MPI_Datatype basic_type = MPI_DATATYPE_NULL; @@ -478,20 +478,6 @@ MPID_Get_accumulate(const void * origin_addr, } MPID_assert(basic_type != MPI_DATATYPE_NULL); - MPI_Datatype result_basic_type = MPI_DATATYPE_NULL; - MPID_Datatype_get_basic_type(result_datatype, result_basic_type); - /* MPID_Datatype_get_basic_type() doesn't handle the struct types */ - if ((result_datatype == MPI_FLOAT_INT) || - (result_datatype == MPI_DOUBLE_INT) || - (result_datatype == MPI_LONG_INT) || - (result_datatype == MPI_SHORT_INT) || - (result_datatype == MPI_LONG_DOUBLE_INT)) - { - MPID_assert(result_basic_type == MPI_DATATYPE_NULL); - result_basic_type = result_datatype; - } - MPID_assert(result_basic_type != MPI_DATATYPE_NULL); - unsigned index; MPIDI_Win_GetAccMsgInfo * headers = MPIU_Calloc0(req->target.dt.num_contig, MPIDI_Win_GetAccMsgInfo); req->accum_headers = headers; @@ -505,11 +491,8 @@ MPID_Get_accumulate(const void * origin_addr, headers[index].count = target_count; headers[index].counter = index; headers[index].num_contig = req->target.dt.num_contig; + headers[index].size = req->target.dt.size; headers[index].request = req; - headers[index].result_addr = result_addr; - headers[index].result_count = result_count; - headers[index].result_datatype = result_basic_type; - headers[index].result_num_contig= req->result_num_contig; } } diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 6969351..180aaad 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -199,11 +199,9 @@ typedef struct int count; int counter; int num_contig; + int size; void * request; void * result_addr; - int result_count; - MPI_Datatype result_datatype; - int result_num_contig; pami_endpoint_t src_endpoint; } MPIDI_Win_GetAccMsgInfo; @@ -245,13 +243,19 @@ typedef struct _mpidi_win_request MPIDI_Datatype dt; } target; + struct + { + void *addr; + int count; + MPI_Datatype datatype; + MPIDI_Datatype dt; + } result; + void *user_buffer; - void *compare_buffer; /* anchor of compare buffer for compare and swap */ + void *compare_buffer; /* anchor of compare buffer for compare and swap */ uint32_t buffer_free; void *buffer; struct _mpidi_win_request *next; - void * compare_addr; - void * result_addr; MPI_Op op; int result_num_contig; http://git.mpich.org/mpich.git/commitdiff/c03e766f497f91de0f75a37c945a8e356… commit c03e766f497f91de0f75a37c945a8e356f1d3788 Author: Su Huang <suhuang(a)us.ibm.com> Date: Mon Jan 6 15:25:55 2014 -0500 PAMID: fix memory leak problem Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/mpid_finalize.c b/src/mpid/pamid/src/mpid_finalize.c index 2ed8633..82da86f 100644 --- a/src/mpid/pamid/src/mpid_finalize.c +++ b/src/mpid/pamid/src/mpid_finalize.c @@ -37,12 +37,19 @@ extern conn_info *_conn_info_list; void MPIDI_close_pe_extension() { + extern MPIDI_printenv_t *mpich_env; + extern MPIX_stats_t *mpid_statp; int rc; /* PAMI_Extension_open in pami_init */ rc = PAMI_Extension_close (pe_extension); if (rc != PAMI_SUCCESS) { TRACE_ERR("ERROR close PAMI_Extension failed rc %d", rc); } + if (mpich_env) + MPIU_Free(mpich_env); + if (mpid_statp) + MPIU_Free(mpid_statp); + } #endif diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c index 70f2a60..1c6cf3c 100644 --- a/src/mpid/pamid/src/mpidi_util.c +++ b/src/mpid/pamid/src/mpidi_util.c @@ -45,7 +45,7 @@ #define PAMI_ASYNC_EXT_ATTR 2000 #if (MPIDI_PRINTENV || MPIDI_STATISTICS || MPIDI_BANNER) -MPIDI_printenv_t *mpich_env; +MPIDI_printenv_t *mpich_env=NULL; extern char* mp_euilib; char mp_euidevice[20]; extern pami_extension_t pe_extension; diff --git a/src/mpid/pamid/src/onesided/mpid_1s.c b/src/mpid/pamid/src/onesided/mpid_1s.c index dd3b234..3b89aab 100644 --- a/src/mpid/pamid/src/onesided/mpid_1s.c +++ b/src/mpid/pamid/src/onesided/mpid_1s.c @@ -54,7 +54,9 @@ MPIDI_Win_DoneCB(pami_context_t context, } - if (req->origin.completed == req->target.dt.num_contig) + if ((req->origin.completed == req->target.dt.num_contig) || + ((req->type >= MPIDI_WIN_REQUEST_COMPARE_AND_SWAP) && + (req->origin.completed == req->origin.dt.num_contig))) { if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); http://git.mpich.org/mpich.git/commitdiff/039c2290692c6fff053fd62ec2a3b5da7… commit 039c2290692c6fff053fd62ec2a3b5da700c7163 Author: Su Huang <suhuang(a)us.ibm.com> Date: Mon Jan 6 14:10:59 2014 -0500 Fix multiple FCNAME declarations error. Also clean up warnings. src/mpid/pamid/src/dyntask/mpidi_port.c:53: warning: useless storage class specifier in empty declaration Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c b/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c index d470475..2b4bd86 100644 --- a/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c +++ b/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c @@ -112,7 +112,6 @@ int MPID_Comm_spawn_multiple(int count, char *array_of_commands[], int array_of_errcodes[]) { int mpi_errno = MPI_SUCCESS; - static char FCNAME[] = "MPID_Comm_spawn_multiple"; if(mpidi_dynamic_tasking == 0) { fprintf(stderr, "Received spawn request for non-dynamic jobs\n"); @@ -359,7 +358,10 @@ int MPIDI_Comm_spawn_multiple(int count, char **commands, static char *parent_port_name = 0; /* Name of parent port if this process was spawned (and is root of comm world) or null */ - +#undef FUNCNAME +#define FUNCNAME MPIDI_GetParentPort +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) int MPIDI_GetParentPort(char ** parent_port) { int mpi_errno = MPI_SUCCESS; diff --git a/src/mpid/pamid/src/dyntask/mpid_port.c b/src/mpid/pamid/src/dyntask/mpid_port.c index b0415b5..e6f8810 100644 --- a/src/mpid/pamid/src/dyntask/mpid_port.c +++ b/src/mpid/pamid/src/dyntask/mpid_port.c @@ -98,7 +98,6 @@ int MPID_Comm_accept(const char * port_name, MPID_Info * info, int root, MPID_Comm * comm, MPID_Comm ** newcomm_ptr) { int mpi_errno = MPI_SUCCESS; - static char FCNAME[] = "MPID_Comm_accept"; if(mpidi_dynamic_tasking == 0) { fprintf(stderr, "Dynamic tasking API is called on non-dynamic jobs\n"); @@ -127,7 +126,6 @@ int MPID_Comm_connect(const char * port_name, MPID_Info * info, int root, MPID_Comm * comm, MPID_Comm ** newcomm_ptr) { int mpi_errno=MPI_SUCCESS; - static char FCNAME[] = "MPID_Comm_connect"; if(mpidi_dynamic_tasking == 0) { fprintf(stderr, "Dynamic tasking API is called on non-dynamic jobs\n"); diff --git a/src/mpid/pamid/src/dyntask/mpidi_port.c b/src/mpid/pamid/src/dyntask/mpidi_port.c index 2d4fecf..fcdbee1 100644 --- a/src/mpid/pamid/src/dyntask/mpidi_port.c +++ b/src/mpid/pamid/src/dyntask/mpidi_port.c @@ -50,7 +50,6 @@ static int maxAcceptQueueSize = 0; static int AcceptQueueSize = 0; pthread_mutex_t rem_connlist_mutex = PTHREAD_MUTEX_INITIALIZER; -extern struct transactionID; /* FIXME: If dynamic processes are not supported, this file will contain no code and some compilers may warn about an "empty translation unit" */ http://git.mpich.org/mpich.git/commitdiff/e25dc31f0ba40a35086b60e76afe1069a… commit e25dc31f0ba40a35086b60e76afe1069af876b58 Author: Su Huang <suhuang(a)us.ibm.com> Date: Tue Dec 31 12:44:13 2013 -0500 PAMID: segfault occurred in MPIDI_Win_DoneCB The segfault occurred while trying to free an already freed request handle in MPIDI_Win_DoneCB. To fix the problem, the second MPIU_Free(req) is removed from the routine. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_1s.c b/src/mpid/pamid/src/onesided/mpid_1s.c index b111546..dd3b234 100644 --- a/src/mpid/pamid/src/onesided/mpid_1s.c +++ b/src/mpid/pamid/src/onesided/mpid_1s.c @@ -69,9 +69,6 @@ MPIDI_Win_DoneCB(pami_context_t context, if (!((req->type > MPIDI_WIN_REQUEST_GET_ACCUMULATE) && (req->type <=MPIDI_WIN_REQUEST_RGET_ACCUMULATE))) MPIU_Free(req); } - - if ( (req->origin.completed == req->origin.dt.num_contig) && ( (req->type == MPIDI_WIN_REQUEST_FETCH_AND_OP) || (req->type == MPIDI_WIN_REQUEST_COMPARE_AND_SWAP) ) ) - MPIU_Free(req); MPIDI_Progress_signal(); } http://git.mpich.org/mpich.git/commitdiff/31ac6f01a51883c466255cd8ae80ae32a… commit 31ac6f01a51883c466255cd8ae80ae32a9d7a6a9 Author: Su Huang <suhuang(a)us.ibm.com> Date: Wed Oct 30 16:30:59 2013 -0400 PAMID: fixes for MPI_Win_* PAMID: fixes for RMA shared related, lock_all and unlock_all functions The following has been updated: - MPID_Win_allocated_shared - MPID_Win_shared_query - several functions in mpid_win_lock_all.c, mpid_win_lock.c, mpid_win_free.c and structures in mpidi_onesided.h, mpidi_datatypes.h have been modified for scalability support for MPID_Win_lock_all() and MPID_Win_unlock_all() PAMID: fix MPID_Win_allocate_shared, MPID_Win_lock_all/unlock_all and MPID_Win_shared_query Add changes based on code review feedback from the team: 1) update getPageSize to ensure that the pagesize is obtained from the range of passed in address. 2) don't call dispatcher if lockQ[index].done == 1 if (!lockQ[index].done) MPID_PROGRESS_WAIT_WHILE(lockQ[index].done == 0); 3) in mpid_win_shared_query() replace MPID_assert(win->create_flavor == MPI_WIN_FLAVOR_SHARED); by MPIU_ERR_CHKANDSTMT((win->create_flavor != MPI_WIN_FLAVOR_SHARED), mpi_errno, MPI_ERR_RMA_FLAVOR, return mpi_errno, "**rmaflavor"); 3) some minor fixes. PAMID: modify MPI_Win_flush_local, MPI_Win_lock etc for better performance The current implementation for the subject mentioned function is to allocate two counters for each rank in the window group. The design could cause a scaling issue. The fix is to update MPI_Win_flush_local etc with two counters per window approach. The changes also include the follwoing: - provides mutex_lock/mutex_unlock for atomic operations in shared window - fixes some bugs in handling shared window. - removes the shared segment with IPC_RMID in MPI_Win_free. PAMID: fixed base address for each rank in a window group For shared window, the base address for each rank should not be exchanged among ranks in a window group. Without the fix, the job will be terminated with segfault. (ibm) F189033 (ibm) D194640 Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 2d98dcf..b773ce7 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -388,6 +388,7 @@ struct MPIDI_Win_lock unsigned rank; MPIDI_LOCK_TYPE_t mtype; /* MPIDI_REQUEST_LOCK or MPIDI_REQUEST_LOCKALL */ int type; + void *flagAddr; }; struct MPIDI_Win_queue { @@ -415,12 +416,6 @@ typedef struct MPIDI_Win_info_args { int alloc_shared_noncontig; } MPIDI_Win_info_args; - -typedef struct { - int nStarted; - int nCompleted; -} RMA_nOps_t; - typedef struct workQ_t { void *msgQ; int count; @@ -446,13 +441,18 @@ typedef struct MPIDI_Win_info uint32_t memregion_used; } MPIDI_Win_info; +typedef pthread_mutex_t MPIDI_SHM_MUTEX; + typedef struct MPIDI_Win_shm_t { int allocated; /* flag: TRUE iff this window has a shared memory region associated with it */ void *base_addr; /* base address of shared memory region */ MPI_Aint segment_len; /* size of shared memory region */ - uint32_t shm_key; /* shared memory key */ + uint32_t shm_id; /* shared memory id */ + int *shm_count; + MPIDI_SHM_MUTEX *mutex_lock; /* shared memory windows -- lock for */ + /* accumulate/atomic operations */ } MPIDI_Win_shm_t; /** @@ -465,7 +465,7 @@ struct MPIDI_Win void ** shm_base_addrs; /* base address shared by all process in comm */ MPIDI_Win_shm_t *shm; /* shared memory info */ workQ_t work; - RMA_nOps_t *origin; + int max_ctrlsends; struct MPIDI_Win_sync { #if 0 diff --git a/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c b/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c index be71200..d470475 100644 --- a/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c +++ b/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c @@ -112,6 +112,7 @@ int MPID_Comm_spawn_multiple(int count, char *array_of_commands[], int array_of_errcodes[]) { int mpi_errno = MPI_SUCCESS; + static char FCNAME[] = "MPID_Comm_spawn_multiple"; if(mpidi_dynamic_tasking == 0) { fprintf(stderr, "Received spawn request for non-dynamic jobs\n"); diff --git a/src/mpid/pamid/src/dyntask/mpid_port.c b/src/mpid/pamid/src/dyntask/mpid_port.c index e6f8810..b0415b5 100644 --- a/src/mpid/pamid/src/dyntask/mpid_port.c +++ b/src/mpid/pamid/src/dyntask/mpid_port.c @@ -98,6 +98,7 @@ int MPID_Comm_accept(const char * port_name, MPID_Info * info, int root, MPID_Comm * comm, MPID_Comm ** newcomm_ptr) { int mpi_errno = MPI_SUCCESS; + static char FCNAME[] = "MPID_Comm_accept"; if(mpidi_dynamic_tasking == 0) { fprintf(stderr, "Dynamic tasking API is called on non-dynamic jobs\n"); @@ -126,6 +127,7 @@ int MPID_Comm_connect(const char * port_name, MPID_Info * info, int root, MPID_Comm * comm, MPID_Comm ** newcomm_ptr) { int mpi_errno=MPI_SUCCESS; + static char FCNAME[] = "MPID_Comm_connect"; if(mpidi_dynamic_tasking == 0) { fprintf(stderr, "Dynamic tasking API is called on non-dynamic jobs\n"); diff --git a/src/mpid/pamid/src/onesided/mpid_1s.c b/src/mpid/pamid/src/onesided/mpid_1s.c index 618f01f..b111546 100644 --- a/src/mpid/pamid/src/onesided/mpid_1s.c +++ b/src/mpid/pamid/src/onesided/mpid_1s.c @@ -56,7 +56,6 @@ MPIDI_Win_DoneCB(pami_context_t context, if (req->origin.completed == req->target.dt.num_contig) { - req->win->mpid.origin[target_rank].nCompleted++; if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); diff --git a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c index d6c3224..7792d8f 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c @@ -251,7 +251,6 @@ MPID_Accumulate(void *origin_addr, MPIU_Free(req); return MPI_SUCCESS; } - win->mpid.origin[target_rank].nStarted++; req->target.rank = target_rank; diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate.c b/src/mpid/pamid/src/onesided/mpid_win_allocate.c index d17ff26..f9ff368 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate.c @@ -86,7 +86,7 @@ MPID_Win_allocate(MPI_Aint size, winfo->win = win; winfo->disp_unit = disp_unit; - rc= MPIDI_Win_allgather(baseP,size,win_ptr); + rc= MPIDI_Win_allgather(size,win_ptr); if (rc != MPI_SUCCESS) return rc; *(void**) base_ptr = (void *) win->base; diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 97525f5..d0f687b 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -36,10 +36,12 @@ extern int mpidi_dynamic_tasking; #define MPIDI_PAGESIZE ((MPI_Aint)pageSize) #define MPIDI_PAGESIZE_MASK (~(MPIDI_PAGESIZE-1)) #define MPIDI_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_PAGESIZE_MASK)) & MPIDI_PAGESIZE_MASK) +#define ALIGN_BOUNDARY 128 /* Align data structures to cache line */ +#define PAD_SIZE(s) (ALIGN_BOUNDARY - (sizeof(s) & (ALIGN_BOUNDARY-1))) int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { - int rank,comm_size; + int rank,comm_size,i; int mpi_errno=PAMI_SUCCESS; @@ -47,9 +49,13 @@ int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { comm_size = comm_ptr->local_size; rank = comm_ptr->rank; - *onNode=0; - if (comm_ptr->intranode_table[rank] != -1) - *onNode=1; + *onNode=1; + for (i=0; i< comm_size; i++) { + if (comm_ptr->intranode_table[i] == -1) { + *onNode=0; + break; + } + } if (*onNode== 0) { MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_CONFLICT, return mpi_errno, "**rmaconflict"); @@ -109,7 +115,7 @@ int GetPageSize(void *addr, ulong *pageSize) while(fgets(Line,200,fp)) { i++; sscanf(Line,"%s %s %s %s \n",A1,A2,A3,A4); - if (memcmp(A1,"KernelPageSize",14)==0) { + if ((found == 1) && (memcmp(A1,"KernelPageSize",14)==0)) { j=atoi(A2); if ((A3[0]=='k') || (A3[0]=='K')) k=1024; @@ -118,13 +124,29 @@ int GetPageSize(void *addr, ulong *pageSize) else if ((A3[0]=='g') || (A3[0]=='G')) k=0x40000000; /* 1 GB */ else { - printf("ERROR unrecognized unit A3=%s\n",A3); + TRACE_ERR("ERROR unrecognized unit A3=%s\n",A3); break; } *pageSize = (ulong)(j * k); TRACE_ERR(" addr=%p pageSize=%ld %s(%d)\n", addr,*pageSize,__FILE__,__LINE__); break; } + if ((strlen(A2) == 4) && ((A2[0]=='r') || (A2[3]=='p'))) { + len = strlen(A1); + #ifndef REDHAT + t1=strtok(A1,search); + #else + t1=strtok(A1,"-"); + #endif + t2 = A1+strlen(t1)+1; + sscanf(t1,"%p \n",&beg); + sscanf(t2,"%p \n",&end); + if (((ulong) addr >= (ulong)beg) && ((ulong)addr <= (ulong)end)) { + found=1; + TRACE_ERR("found addr=%p i=%d between beg=%p and end=%p in %s\n", + addr,i,beg,end,fileName); + } + } } fclose(fp); if (*pageSize == 0) { @@ -135,34 +157,28 @@ int GetPageSize(void *addr, ulong *pageSize) return 0; } -#define MPIDI_PAGESIZE ((MPI_Aint)pageSize) -#define MPIDI_PAGESIZE_MASK (~(MPIDI_PAGESIZE-1)) -#define MPIDI_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_PAGESIZE_MASK)) & MPIDI_PAGESIZE_MASK) - - - int -MPID_getSharedSegment(MPI_Aint size, +MPID_getSharedSegment(MPI_Aint size, int disp_unit, - MPID_Info * info, MPID_Comm * comm_ptr, - void **base_ptr, - MPID_Win ** win_ptr) + void **base_ptr, + MPID_Win **win_ptr, + MPI_Aint *pSize, + int *noncontig) { int mpi_errno = MPI_SUCCESS; - void **base_pp = (void **) base_ptr; + void **base_pp = base_ptr; int i, k, comm_size, rank; - int shm_id; uint32_t shm_key; int node_rank; + int shm_id; MPI_Aint *node_sizes; - void * base_addr; MPI_Aint *tmp_buf; int errflag = FALSE; - int noncontig = FALSE; - MPI_Aint pageSize, len,new_size; + MPI_Aint pageSize,pageSize2, len,new_size; char *cp; MPID_Win *win; + int padSize; MPIDI_Win_info *winfo; int shm_flag = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR; @@ -171,14 +187,14 @@ MPID_getSharedSegment(MPI_Aint size, rank = win->comm_ptr->rank; tmp_buf = MPIU_Malloc( 2*comm_size*sizeof(MPI_Aint)); - mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); if (mpi_errno) MPIU_ERR_POP(mpi_errno); - GetPageSize((void *) win_ptr, &pageSize); + GetPageSize((void *) win_ptr, (ulong *) &pageSize); + *pSize = pageSize; win->mpid.shm->segment_len = 0; if (comm_size == 1) { if (size > 0) { - if (noncontig) + if (*noncontig) new_size = MPIDI_ROUND_UP_PAGESIZE(size); else new_size = size; @@ -196,6 +212,7 @@ MPID_getSharedSegment(MPI_Aint size, if (mpi_errno) MPIU_ERR_POP(mpi_errno); win->mpid.shm->segment_len = new_size; win->mpid.info[rank].base_addr = *base_pp; + win->base = *base_pp; } else { tmp_buf[rank] = (MPI_Aint) size; mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, @@ -206,12 +223,13 @@ MPID_getSharedSegment(MPI_Aint size, /* calculate total number of bytes needed */ for (i = 0; i < comm_size; ++i) { len = tmp_buf[i]; - if (noncontig) + if (*noncontig) /* Round up to next page size */ win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(len); else win->mpid.shm->segment_len += len; } + len = len + 128; /* needed for mutex_lock etc */ /* get shared segment */ shm_key=-1; @@ -266,7 +284,13 @@ MPID_getSharedSegment(MPI_Aint size, shm_id = shmget(shm_key, win->mpid.shm->segment_len, shm_flag); MPIU_ERR_CHKANDJUMP((shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); win->mpid.shm->base_addr = (void *) shmat(shm_id,0,0); - MPIU_ERR_CHKANDJUMP((base_addr == NULL), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); + MPIU_ERR_CHKANDJUMP((win->mpid.shm->base_addr == NULL), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); + GetPageSize((void *) win->mpid.shm->base_addr, &pageSize2); + MPID_assert(pageSize == pageSize2); + /* set mutex_lock address and initialize it */ + win->mpid.shm->mutex_lock = (pthread_mutex_t *) win->mpid.shm->base_addr; + win->mpid.shm->shm_count=(int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(pthread_mutex_t)); + MPIDI_SHM_MUTEX_INIT(win); win->mpid.shm->allocated = 1; /* successfully created shm segment */ mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, comm_ptr, &errflag); @@ -278,35 +302,22 @@ MPID_getSharedSegment(MPI_Aint size, if (shm_id != -1) { /* shm segment is available */ win->mpid.shm->base_addr = (void *) shmat(shm_id,0,0); win->mpid.shm->allocated = 1; - MPIU_ERR_CHKANDJUMP((base_addr == (void *) -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + MPIU_ERR_CHKANDJUMP((win->mpid.shm->base_addr == (void *) -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); } else { /* node leader failed, no need to try here */ MPIU_ERR_CHKANDJUMP((shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); } + win->mpid.shm->mutex_lock = (pthread_mutex_t *) win->mpid.shm->base_addr; + win->mpid.shm->shm_count=(int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(pthread_mutex_t)); } - mpi_errno = MPIR_Barrier_impl(comm_ptr, &errflag); + win->mpid.shm->shm_id = shm_id; + OPA_fetch_and_add_int((OPA_int_t *) win->mpid.shm->shm_count,1); + while(*win->mpid.shm->shm_count != comm_size) MPIDI_QUICKSLEEP; /* wait for all ranks complete shmat */ /* compute the base addresses of each process within the shared memory segment */ { - win->base = win->mpid.shm->base_addr; - winfo = &win->mpid.info[rank]; - winfo->win = win; - winfo->disp_unit = disp_unit; - win->mpid.info[0].base_addr = win->mpid.shm->base_addr; - char *cur_base = (*win_ptr)->mpid.shm->base_addr; - for (i = 1; i < comm_size; ++i) { - size = tmp_buf[i]; - if (size) { - if (noncontig) - /* Round up to next page size */ - win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size)); - else - win->mpid.info[i].base_addr = (void *) ((MPI_Aint) cur_base + size); - cur_base = win->mpid.info[i].base_addr; - } else { - win->mpid.info[i].base_addr = NULL; - } - } - } - *base_pp = win->mpid.info[rank].base_addr; + padSize=sizeof(pthread_mutex_t) + sizeof(OPA_int_t); + win->base = (void *) ((long) win->mpid.shm->base_addr + (long ) PAD_SIZE(padSize)); + } + *base_pp = win->base; } fn_exit: @@ -352,6 +363,11 @@ fn_fail: * \param[out] win_ptr window object returned by the call (handle) * \return MPI_SUCCESS, MPI_ERR_ARG, MPI_ERR_COMM, MPI_ERR_INFO. MPI_ERR_OTHER, * MPI_ERR_SIZE + * + * win->mpid.shm->base_addr \* return address from shmat *\ + * win->base \* address for data starts here == win->mpid.shm->base_addr *\ + * \* + space for mutex_lock and shm_count *\ + * */ int MPID_Win_allocate_shared(MPI_Aint size, @@ -365,35 +381,49 @@ MPID_Win_allocate_shared(MPI_Aint size, void **baseP = base_ptr; MPIDI_Win_info *winfo; MPID_Win *win; - int rank, comm_size; - int onNode; + int rank, comm_size,i; + int onNode,noncontig=FALSE; + MPI_Aint pageSize=0; + mpi_errno =MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_SHARED, MPI_WIN_UNIFIED); if (mpi_errno) MPIU_ERR_POP(mpi_errno); win = *win_ptr; mpi_errno=CheckRankOnNode(comm_ptr,&onNode); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_ERR_CHKANDJUMP((onNode == 0), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - + mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); rank = (*win_ptr)->comm_ptr->rank; comm_size = (*win_ptr)->comm_ptr->local_size; win->mpid.shm = MPIU_Malloc(sizeof(MPIDI_Win_shm_t)); win->mpid.shm->allocated=0; MPID_assert(win->mpid.shm != NULL); - MPID_getSharedSegment(size, disp_unit,info,comm_ptr,baseP, win_ptr); + MPID_getSharedSegment(size, disp_unit,comm_ptr,baseP, win_ptr,(ulong *)&pageSize,(int *)&noncontig); - win->base = *baseP; winfo = &win->mpid.info[rank]; winfo->win = win; winfo->disp_unit = disp_unit; - win->base = (void *) MPIU_PtrToAint(winfo->base_addr); - winfo->base_addr = win->base; - - mpi_errno = MPIDI_Win_allgather(*baseP,size,win_ptr); + mpi_errno = MPIDI_Win_allgather(size,win_ptr); if (mpi_errno != MPI_SUCCESS) return mpi_errno; - - *(void**) base_ptr = (void *) win->base; + win->mpid.info[0].base_addr = win->base; + if (comm_size > 1) { + char *cur_base = (*win_ptr)->base; + for (i = 1; i < comm_size; ++i) { + if (size) { + if (noncontig) + /* Round up to next page size */ + win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size)); + else + win->mpid.info[i].base_addr = (void *) ((MPI_Aint) cur_base + (MPI_Aint) size); + cur_base = win->mpid.info[i].base_addr; + } else { + win->mpid.info[i].base_addr = NULL; + } + } + } + *(void**) base_ptr = (void *) win->mpid.info[rank].base_addr; mpi_errno = MPIR_Barrier_impl(comm_ptr, &mpi_errno); diff --git a/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c b/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c index 8022bc7..3f7d925 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c +++ b/src/mpid/pamid/src/onesided/mpid_win_compare_and_swap.c @@ -112,7 +112,6 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, int disp_unit; int len; -#ifdef PENDING_SHM_WIN if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) { MPIDI_SHM_MUTEX_LOCK(win); shm_locked = 1; @@ -121,12 +120,9 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, disp_unit = win->disp_unit; } else { -#endif base = win->base; disp_unit = win->disp_unit; -#ifdef PENDING_SHM_WIN } -#endif dest_addr = (char *) base + disp_unit * target_disp; @@ -136,12 +132,10 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, if (MPIR_Compare_equal(compare_addr, dest_addr, datatype)) MPIU_Memcpy(dest_addr, origin_addr, len); -#ifdef PENDING_SHM_WIN if (shm_locked) { MPIDI_SHM_MUTEX_UNLOCK(win); shm_locked = 0; } -#endif MPIU_Free(req); } else { diff --git a/src/mpid/pamid/src/onesided/mpid_win_create.c b/src/mpid/pamid/src/onesided/mpid_win_create.c index 38f072a..e3d2b9d 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_create.c +++ b/src/mpid/pamid/src/onesided/mpid_win_create.c @@ -78,7 +78,6 @@ MPIDI_Win_init( MPI_Aint length, MPID_assert(mpi_errno == 0); } MPID_assert(mpi_errno == 0); - win->mpid.origin = MPIU_Calloc0(size, RMA_nOps_t); /* Initialize the info (hint) flags per window */ @@ -106,7 +105,7 @@ MPIDI_Win_init( MPI_Aint length, /* */ /***************************************************************************/ int -MPIDI_Win_allgather(void *base, MPI_Aint size, MPID_Win **win_ptr ) +MPIDI_Win_allgather( MPI_Aint size, MPID_Win **win_ptr ) { int mpi_errno = MPI_SUCCESS; MPID_Win *win; @@ -208,7 +207,7 @@ MPID_Win_create(void * base, winfo->win = win; winfo->disp_unit = disp_unit; - rc= MPIDI_Win_allgather(base,size,win_ptr); + rc= MPIDI_Win_allgather(size,win_ptr); if (rc != MPI_SUCCESS) return rc; diff --git a/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c b/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c index 5343362..74e10ca 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c +++ b/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c @@ -57,7 +57,7 @@ MPID_Win_create_dynamic( MPID_Info * info, winfo = &win->mpid.info[rank]; winfo->win = win; - rc= MPIDI_Win_allgather(MPI_BOTTOM,0,win_ptr); + rc= MPIDI_Win_allgather(0,win_ptr); if (rc != MPI_SUCCESS) return rc; diff --git a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c index 153ef88..9f47b9b 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c +++ b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c @@ -293,7 +293,6 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, int disp_unit; int len, one; -#ifdef PENDING_SHM_WIN if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) { MPIDI_SHM_MUTEX_LOCK(win); shm_locked = 1; @@ -302,12 +301,9 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, } else { -#endif base = win->base; disp_unit = win->disp_unit; -#ifdef PENDING_SHM_WIN } -#endif dest_addr = (char *) base + disp_unit * target_disp; @@ -319,12 +315,10 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, (*uop)((void *) origin_addr, dest_addr, &one, &datatype); -#ifdef PENDING_SHM_WIN if (shm_locked) { MPIDI_SHM_MUTEX_UNLOCK(win); shm_locked = 0; } -#endif MPIU_Free(req); diff --git a/src/mpid/pamid/src/onesided/mpid_win_flush.c b/src/mpid/pamid/src/onesided/mpid_win_flush.c index 4fb3dde..c2e7e29 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_flush.c +++ b/src/mpid/pamid/src/onesided/mpid_win_flush.c @@ -52,12 +52,11 @@ MPID_Win_flush(int rank, MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_SYNC, return mpi_errno, "**rmasync"); } - MPID_PROGRESS_WAIT_WHILE(win->mpid.origin[rank].nStarted != win->mpid.origin[rank].nCompleted); - sync = &win->mpid.sync; - win->mpid.origin[rank].nStarted=0; - win->mpid.origin[rank].nCompleted=0; - + MPID_PROGRESS_WAIT_WHILE(sync->total != sync->complete); + sync->total = 0; + sync->started = 0; + sync->complete = 0; return mpi_errno; } @@ -80,7 +79,6 @@ int MPID_Win_flush_all(MPID_Win *win) { int mpi_errno = MPI_SUCCESS; - int nTasks,i; struct MPIDI_Win_sync* sync; static char FCNAME[] = "MPID_Win_flush_all"; @@ -91,15 +89,11 @@ MPID_Win_flush_all(MPID_Win *win) return mpi_errno, "**rmasync"); } - sync = &win->mpid.sync; - MPID_PROGRESS_WAIT_WHILE(sync->total != sync->complete); - sync->total = 0; - sync->started = 0; - sync->complete = 0; - for (i = 0; i < MPIR_Comm_size(win->comm_ptr); i++) { - win->mpid.origin[i].nStarted=0; - win->mpid.origin[i].nCompleted=0; - } + sync = &win->mpid.sync; + MPID_PROGRESS_WAIT_WHILE(sync->total != sync->complete); + sync->total = 0; + sync->started = 0; + sync->complete = 0; return mpi_errno; } @@ -114,6 +108,10 @@ MPID_Win_flush_all(MPID_Win *win) * the cng process to the target rank on the given window. The user may * reuse any buffers after this routine returns. * + * It has been determined that the routine uses only counters for each window + * and not for each rank because the overhead of tracking each rank could be great + * if a window group contains a large number of ranks. + * * \param[in] rank rank of target window * \param[in] win window object * \return MPI_SUCCESS, MPI_ERR_OTHER @@ -123,6 +121,7 @@ int MPID_Win_flush_local(int rank, MPID_Win *win) { int mpi_errno = MPI_SUCCESS; + struct MPIDI_Win_sync* sync; static char FCNAME[] = "MPID_Win_flush_local"; if((win->mpid.sync.origin_epoch_type != MPID_EPOTYPE_LOCK) && @@ -131,9 +130,11 @@ MPID_Win_flush_local(int rank, MPID_Win *win) MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_SYNC, return mpi_errno, "**rmasync"); } - MPID_PROGRESS_WAIT_WHILE(win->mpid.origin[rank].nStarted != win->mpid.origin[rank].nCompleted); - win->mpid.origin[rank].nStarted=0; - win->mpid.origin[rank].nCompleted=0; + sync = &win->mpid.sync; + MPID_PROGRESS_WAIT_WHILE(sync->total != sync->complete); + sync->total = 0; + sync->started = 0; + sync->complete = 0; return mpi_errno; } @@ -155,8 +156,8 @@ int MPID_Win_flush_local_all(MPID_Win *win) { int mpi_errno = MPI_SUCCESS; + struct MPIDI_Win_sync* sync; static char FCNAME[] = "MPID_Win_flush"; - int size,i; if((win->mpid.sync.origin_epoch_type != MPID_EPOTYPE_LOCK) && (win->mpid.sync.origin_epoch_type != MPID_EPOTYPE_LOCK_ALL)) @@ -164,14 +165,11 @@ MPID_Win_flush_local_all(MPID_Win *win) MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_SYNC, return mpi_errno, "**rmasync"); } - size = MPIR_Comm_size(win->comm_ptr); - for (i=0; i < size; i++) { - MPID_PROGRESS_WAIT_WHILE(win->mpid.origin[i].nStarted != win->mpid.origin[i].nCompleted); - } - for (i=0; i < size; i++) { - win->mpid.origin[i].nStarted=0; - win->mpid.origin[i].nCompleted=0; - } + sync = &win->mpid.sync; + MPID_PROGRESS_WAIT_WHILE(sync->total != sync->complete); + sync->total = 0; + sync->started = 0; + sync->complete = 0; return mpi_errno; } diff --git a/src/mpid/pamid/src/onesided/mpid_win_free.c b/src/mpid/pamid/src/onesided/mpid_win_free.c index 2ba83d9..c8a8867 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_free.c +++ b/src/mpid/pamid/src/onesided/mpid_win_free.c @@ -20,16 +20,34 @@ * \brief ??? */ #include "mpidi_onesided.h" +#include <sys/shm.h> +#include <sys/ipc.h> +#include <sys/stat.h> + int MPIDI_SHM_Win_free(MPID_Win **win_ptr) { + static char FCNAME[] = "MPID_SHM_Win_free"; + int rc; int mpi_errno = MPI_SUCCESS; + /* Free shared memory region */ /* free shm_base_addrs that's only used for shared memory windows */ - if ((*win_ptr)->mpid.shm->allocated) - mpi_errno = shmdt((*win_ptr)->base); + if ((*win_ptr)->mpid.shm->allocated) { + OPA_fetch_and_add_int((OPA_int_t *) (*win_ptr)->mpid.shm->shm_count,-1); + while(*(*win_ptr)->mpid.shm->shm_count) MPIDI_QUICKSLEEP; + if ((*win_ptr)->comm_ptr->rank == 0) { + MPIDI_SHM_MUTEX_DESTROY(*win_ptr); + } + mpi_errno = shmdt((*win_ptr)->mpid.shm->base_addr); + if ((*win_ptr)->comm_ptr->rank == 0) { + rc=shmctl((*win_ptr)->mpid.shm->shm_id,IPC_RMID,NULL); + MPIU_ERR_CHKANDJUMP((rc == -1), errno,MPI_ERR_RMA_SHARED, "**shmctl"); + } + } MPIU_Free((*win_ptr)->mpid.shm); (*win_ptr)->mpid.shm = NULL; + fn_fail: return mpi_errno; } @@ -62,10 +80,12 @@ MPID_Win_free(MPID_Win **win_ptr) mpi_errno = MPIR_Barrier_impl(win->comm_ptr, &errflag); MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**mpi_bcast"); - - if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) + + if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) mpi_errno=MPIDI_SHM_Win_free(win_ptr); + + struct MPIDI_Win_info *winfo = &win->mpid.info[rank]; #ifdef USE_PAMI_RDMA if (win->size != 0) @@ -82,12 +102,10 @@ MPID_Win_free(MPID_Win **win_ptr) MPID_assert(rc == PAMI_SUCCESS); } #endif + MPIU_Free(win->mpid.info); - MPIU_Free(win->mpid.origin); if (win->mpid.work.msgQ) MPIU_Free(win->mpid.work.msgQ); - if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) - MPIU_Free(win->base); MPIR_Comm_release(win->comm_ptr, 0); diff --git a/src/mpid/pamid/src/onesided/mpid_win_get.c b/src/mpid/pamid/src/onesided/mpid_win_get.c index 0d5b6a2..b56e2b2 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get.c @@ -258,13 +258,11 @@ MPID_Get(void *origin_addr, MPIU_Free(req); return MPI_SUCCESS; } - win->mpid.origin[target_rank].nStarted++; /* If the get is a local operation, do it here */ if (target_rank == win->comm_ptr->rank) { size_t offset = req->offset; - win->mpid.origin[target_rank].nCompleted++; if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); else diff --git a/src/mpid/pamid/src/onesided/mpid_win_lock.c b/src/mpid/pamid/src/onesided/mpid_win_lock.c index 259ab75..5871ddc 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_lock.c +++ b/src/mpid/pamid/src/onesided/mpid_win_lock.c @@ -91,8 +91,10 @@ MPIDI_WinLockReq_proc(pami_context_t context, struct MPIDI_Win_lock* lock = MPIU_Calloc0(1, struct MPIDI_Win_lock); if (info->type == MPIDI_WIN_MSGTYPE_LOCKREQ) lock->mtype = MPIDI_REQUEST_LOCK; - else if (info->type == MPIDI_WIN_MSGTYPE_LOCKALLREQ) + else if (info->type == MPIDI_WIN_MSGTYPE_LOCKALLREQ) { lock->mtype = MPIDI_REQUEST_LOCKALL; + lock->flagAddr = (void *) info->flagAddr; + } lock->rank = info->rank; lock->type = info->data.lock.type; @@ -206,9 +208,10 @@ MPID_Win_unlock(int rank, } if (rank == MPI_PROC_NULL) goto fn_exit; struct MPIDI_Win_sync* sync = &win->mpid.sync; - MPID_PROGRESS_WAIT_DO_WHILE(win->mpid.origin[rank].nStarted != win->mpid.origin[rank].nCompleted); - win->mpid.origin[rank].nCompleted=0; - win->mpid.origin[rank].nStarted=0; + MPID_PROGRESS_WAIT_DO_WHILE(sync->total != sync->complete); + sync->total = 0; + sync->started = 0; + sync->complete = 0; MPIDI_WinLock_info info = { .done = 0, diff --git a/src/mpid/pamid/src/onesided/mpid_win_lock_all.c b/src/mpid/pamid/src/onesided/mpid_win_lock_all.c index 74248e4..5fdf67f 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_lock_all.c +++ b/src/mpid/pamid/src/onesided/mpid_win_lock_all.c @@ -29,6 +29,7 @@ MPIDI_WinLockAllReq_post(pami_context_t context, MPIDI_WinLock_info* info = (MPIDI_WinLock_info*)_info; MPIDI_Win_control_t msg = { .type = MPIDI_WIN_MSGTYPE_LOCKALLREQ, + .flagAddr = info, .data = { .lock = { .type = info->lock_type, @@ -42,19 +43,17 @@ MPIDI_WinLockAllReq_post(pami_context_t context, static pami_result_t -MPIDI_WinUnlockAll_post(pami_context_t context, +MPIDI_WinUnlockAllReq_post(pami_context_t context, void * _info) { MPID_Win *win; MPIDI_WinLock_info* info = (MPIDI_WinLock_info*)_info; MPIDI_Win_control_t msg = { .type = MPIDI_WIN_MSGTYPE_UNLOCKALL, + .flagAddr = info, }; win=info->win; MPIDI_WinCtrlSend(context, &msg, info->peer, info->win); - /* current request is the last one in the group */ - if (win->mpid.sync.lock.remote.allLocked == 1) - info->done = 1; return PAMI_SUCCESS; } @@ -88,6 +87,8 @@ MPID_Win_lock_all(int assert, int mpi_errno = MPI_SUCCESS; int i,size; MPIDI_WinLock_info *lockQ; + char *cp=NULL; + int nMask,index; struct MPIDI_Win_sync_lock* slock = &win->mpid.sync.lock; static char FCNAME[] = "MPID_Win_lock_all"; @@ -97,19 +98,47 @@ MPID_Win_lock_all(int assert, return mpi_errno, "**rmasync"); } size = (MPIR_Comm_size(win->comm_ptr)); + win->mpid.max_ctrlsends = MAX_NUM_CTRLSEND; + nMask= win->mpid.max_ctrlsends - 1; + if (cp=getenv("MP_MAX_NUM_CTRLSEND")) { + win->mpid.max_ctrlsends = atoi(cp); + } + nMask=(win->mpid.max_ctrlsends - 1); if (!win->mpid.work.msgQ) { - win->mpid.work.msgQ = (void *) MPIU_Calloc0(size, MPIDI_WinLock_info); + if (size < (win->mpid.max_ctrlsends)) { + win->mpid.work.msgQ = (void *) MPIU_Calloc0(size, MPIDI_WinLock_info); + } else { + win->mpid.work.msgQ = (void *) MPIU_Calloc0((win->mpid.max_ctrlsends), MPIDI_WinLock_info); + } MPID_assert(win->mpid.work.msgQ != NULL); - win->mpid.work.count=0; + win->mpid.work.count=0; } lockQ = (MPIDI_WinLock_info *) win->mpid.work.msgQ; - for (i = 0; i < size; i++) { - lockQ[i].done=0; - lockQ[i].peer=i; - lockQ[i].win=win; - lockQ[i].lock_type=MPI_LOCK_SHARED; - MPIDI_Context_post(MPIDI_Context[0], &lockQ[i].work, MPIDI_WinLockAllReq_post, &lockQ[i]); - } + if (size < win->mpid.max_ctrlsends) { + for (i = 0; i < size; i++) { + lockQ[i].done=0; + lockQ[i].peer=i; + lockQ[i].win=win; + lockQ[i].lock_type=MPI_LOCK_SHARED; + MPIDI_Context_post(MPIDI_Context[0], &lockQ[i].work, MPIDI_WinLockAllReq_post, &lockQ[i]); + } + } else { + for (i = 0; i < size; i++) { + if (i < win->mpid.max_ctrlsends) + index=i; + else { + index = i & nMask; + if (!lockQ[index].done) { + MPID_PROGRESS_WAIT_WHILE(lockQ[index].done == 0); + } + } + lockQ[index].done=0; + lockQ[index].peer=i; + lockQ[index].win=win; + lockQ[index].lock_type=MPI_LOCK_SHARED; + MPIDI_Context_post(MPIDI_Context[0], &lockQ[index].work, MPIDI_WinLockAllReq_post, &lockQ[index]); + } + } /* wait for the lock is granted for all tasks in the window */ MPID_PROGRESS_WAIT_WHILE(size != slock->remote.allLocked); @@ -123,8 +152,9 @@ int MPID_Win_unlock_all(MPID_Win *win) { int mpi_errno = MPI_SUCCESS; - int i; + int i,size; MPIDI_WinLock_info *lockQ; + int nMask,index; struct MPIDI_Win_sync* sync; static char FCNAME[] = "MPID_Win_unlock_all"; @@ -138,17 +168,32 @@ MPID_Win_unlock_all(MPID_Win *win) sync->total = 0; sync->started = 0; sync->complete = 0; - for (i = 0; i < MPIR_Comm_size(win->comm_ptr); i++) { - win->mpid.origin[i].nStarted=0; - win->mpid.origin[i].nCompleted=0; - } MPID_assert(win->mpid.work.msgQ != NULL); lockQ = (MPIDI_WinLock_info *) win->mpid.work.msgQ; - for (i = 0; i < MPIR_Comm_size(win->comm_ptr); i++) { - lockQ[i].done=0; - lockQ[i].peer=i; - lockQ[i].win=win; - MPIDI_Context_post(MPIDI_Context[0], &lockQ[i].work, MPIDI_WinUnlockAll_post, &lockQ[i]); + size = MPIR_Comm_size(win->comm_ptr); + nMask = (win->mpid.max_ctrlsends - 1); + if (size < win->mpid.max_ctrlsends) { + for (i = 0; i < size; i++) { + lockQ[i].done=0; + lockQ[i].peer=i; + lockQ[i].win=win; + MPIDI_Context_post(MPIDI_Context[0], &lockQ[i].work, MPIDI_WinUnlockAllReq_post, &lockQ[i]); + } + } else { + for (i = 0; i < size; i++) { + if (i < win->mpid.max_ctrlsends) + index=i; + else { + index = (i & nMask); + if (!lockQ[index].done) { + MPID_PROGRESS_WAIT_WHILE(lockQ[index].done == 0); + } + } + lockQ[index].done=0; + lockQ[index].peer=i; + lockQ[index].win=win; + MPIDI_Context_post(MPIDI_Context[0], &lockQ[index].work, MPIDI_WinUnlockAllReq_post, &lockQ[index]); + } } MPID_PROGRESS_WAIT_WHILE(sync->lock.remote.allLocked); diff --git a/src/mpid/pamid/src/onesided/mpid_win_put.c b/src/mpid/pamid/src/onesided/mpid_win_put.c index 6e8d38b..67f9d84 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_put.c +++ b/src/mpid/pamid/src/onesided/mpid_win_put.c @@ -274,13 +274,11 @@ MPID_Put(void *origin_addr, return MPI_SUCCESS; } - win->mpid.origin[target_rank].nStarted++; /* If the get is a local operation, do it here */ if (target_rank == win->comm_ptr->rank) { size_t offset = req->offset; - win->mpid.origin[target_rank].nCompleted++; if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); else diff --git a/src/mpid/pamid/src/onesided/mpid_win_shared_query.c b/src/mpid/pamid/src/onesided/mpid_win_shared_query.c index fb304ee..21e70fd 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_shared_query.c +++ b/src/mpid/pamid/src/onesided/mpid_win_shared_query.c @@ -40,8 +40,10 @@ MPID_Win_shared_query(MPID_Win *win, int rank, MPI_Aint *size, int *disp_unit, void *base_ptr) { int mpi_errno = MPI_SUCCESS; - - *(void**) base_ptr = (void *) win->base; + static char FCNAME[] = "MPID_Win_shared_query"; + MPIU_ERR_CHKANDSTMT((win->create_flavor != MPI_WIN_FLAVOR_SHARED), mpi_errno, + MPI_ERR_RMA_FLAVOR, return mpi_errno, "**rmaflavor"); + *((void **) base_ptr) = (void *) win->base; *size = win->size; *disp_unit = win->disp_unit; diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 57aabca..6969351 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -20,6 +20,7 @@ * \brief ??? */ #include <mpidimpl.h> +#include "opa_primitives.h" #ifndef __src_onesided_mpidi_onesided_h__ #define __src_onesided_mpidi_onesided_h__ @@ -33,6 +34,54 @@ pami_send_immediate_t zero_send_immediate_parms; pami_recv_t zero_recv_parms; pami_rmw_t zero_rmw_parms; +#define MPIDI_QUICKSLEEP usleep(1); +#define MAX_NUM_CTRLSEND 1024 /* no more than 1024 outstanding control sends */ + + +#define MPIDI_SHM_MUTEX_LOCK(win) \ + do { \ + pthread_mutex_t *shm_mutex = win->mpid.shm->mutex_lock; \ + int rval = pthread_mutex_lock(shm_mutex); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_lock", \ + "**pthread_lock %s", strerror(rval)); \ + } while (0) + +#define MPIDI_SHM_MUTEX_UNLOCK(win) \ + do { \ + pthread_mutex_t *shm_mutex = win->mpid.shm->mutex_lock; \ + int rval = pthread_mutex_unlock(shm_mutex); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_unlock", \ + "**pthread_unlock %s", strerror(rval)); \ + } while (0) + +#define MPIDI_SHM_MUTEX_INIT(win) \ + do { \ + int rval=0; \ + pthread_mutexattr_t attr; \ + pthread_mutex_t *shm_mutex = win->mpid.shm->mutex_lock; \ + \ + rval = pthread_mutexattr_init(&attr); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_mutex", \ + "**pthread_mutex %s", strerror(rval)); \ + rval = pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_mutex", \ + "**pthread_mutex %s", strerror(rval)); \ + rval = pthread_mutex_init(shm_mutex, &attr); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_mutex", \ + "**pthread_mutex %s", strerror(rval)); \ + rval = pthread_mutexattr_destroy(&attr); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_mutex", \ + "**pthread_mutex %s", strerror(rval)); \ + } while (0); + +#define MPIDI_SHM_MUTEX_DESTROY(win) \ + do { \ + pthread_mutex_t *shm_mutex = (win)->mpid.shm->mutex_lock; \ + int rval = pthread_mutex_destroy(shm_mutex); \ + MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_mutex", \ + "**pthread_mutex %s", strerror(rval)); \ + } while (0); + /** * \brief One-sided Message Types */ @@ -75,6 +124,7 @@ typedef struct MPIDI_Win_msgtype_t type; MPID_Win *win; int rank; /* MPI rank */ + void *flagAddr; union { struct @@ -104,6 +154,7 @@ typedef struct MPIDI_WinLock_info unsigned peer; int lock_type; struct MPID_Win * win; + void *flagAddr; volatile unsigned done; pami_work_t work; } MPIDI_WinLock_info; @@ -280,8 +331,7 @@ MPIDI_Win_init( MPI_Aint length, int create_flavor, int model); int -MPIDI_Win_allgather(void *base, - MPI_Aint size, +MPIDI_Win_allgather(MPI_Aint size, MPID_Win **win_ptr); void diff --git a/src/mpid/pamid/src/onesided/mpidi_win_control.c b/src/mpid/pamid/src/onesided/mpidi_win_control.c index 8c537ff..56016be 100644 --- a/src/mpid/pamid/src/onesided/mpidi_win_control.c +++ b/src/mpid/pamid/src/onesided/mpidi_win_control.c @@ -29,6 +29,7 @@ MPIDI_WinCtrlSend(pami_context_t context, MPID_Win *win) { pami_task_t taskid; + MPIDI_WinLock_info *winLock; control->win = win->mpid.info[rank].win; control->rank = win->comm_ptr->rank; taskid=MPID_VCR_GET_LPID(win->comm_ptr->vcr,rank); @@ -72,6 +73,10 @@ MPIDI_WinCtrlSend(pami_context_t context, }; rc = PAMI_Send_immediate(context, &params); } + if ((control->type == MPIDI_WIN_MSGTYPE_LOCKALLREQ) || (control->type == MPIDI_WIN_MSGTYPE_UNLOCKALL)) { + winLock = (MPIDI_WinLock_info *) control->flagAddr; + winLock->done = 1; + } MPID_assert(rc == PAMI_SUCCESS); } http://git.mpich.org/mpich.git/commitdiff/c6d910c783380440d1946ab366e5f2496… commit c6d910c783380440d1946ab366e5f2496eaed042 Author: Su Huang <suhuang(a)us.ibm.com> Date: Wed Oct 16 14:27:49 2013 -0400 PAMID: MPID_Win_allocate_shared and MPID_Win_shared_query support added MPID_Win_allocate_shared and MPID_Win_shared_query support and fixes on passing incorrect rank to MPIDI_WinCtrlSend in MPIDI_WinPost_post and MPIDI_WinComplete_post minor fixes for 'win allocate shared' - compiles for bgq - UNRESOLVED: undefined shm key on bgq, or any other time the required environment variables are not set - UNRESOLVED: multiple calls to 'win allocate shared' will use the same shm key (ibm) F189033 Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 203961e..2d98dcf 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -445,6 +445,16 @@ typedef struct MPIDI_Win_info pami_memregion_t memregion; /**< Memory region descriptor for each node */ uint32_t memregion_used; } MPIDI_Win_info; + +typedef struct MPIDI_Win_shm_t +{ + int allocated; /* flag: TRUE iff this window has a shared memory + region associated with it */ + void *base_addr; /* base address of shared memory region */ + MPI_Aint segment_len; /* size of shared memory region */ + uint32_t shm_key; /* shared memory key */ +} MPIDI_Win_shm_t; + /** * \brief Structure of PAMI extensions to MPID_Win structure */ @@ -453,7 +463,8 @@ struct MPIDI_Win struct MPIDI_Win_info *info; /**< allocated array of collective info */ MPIDI_Win_info_args info_args; void ** shm_base_addrs; /* base address shared by all process in comm */ - workQ_t work; + MPIDI_Win_shm_t *shm; /* shared memory info */ + workQ_t work; RMA_nOps_t *origin; struct MPIDI_Win_sync { diff --git a/src/mpid/pamid/src/misc/mpid_unimpl.c b/src/mpid/pamid/src/misc/mpid_unimpl.c index 44bda50..1009566 100644 --- a/src/mpid/pamid/src/misc/mpid_unimpl.c +++ b/src/mpid/pamid/src/misc/mpid_unimpl.c @@ -90,17 +90,3 @@ int MPID_Comm_group_failed(MPID_Comm *comm_ptr, MPID_Group **failed_group_ptr) MPID_abort(); return 0; } - -int MPID_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info_ptr, MPID_Comm *comm_ptr, - void **base_ptr, MPID_Win **win_ptr) -{ - MPID_abort(); - return 0; -} - -int MPID_Win_shared_query(MPID_Win *win, int rank, MPI_Aint *size, int *disp_unit, - void *baseptr) -{ - MPID_abort(); - return 0; -} diff --git a/src/mpid/pamid/src/onesided/Makefile.mk b/src/mpid/pamid/src/onesided/Makefile.mk index cb526db..9a6cd37 100644 --- a/src/mpid/pamid/src/onesided/Makefile.mk +++ b/src/mpid/pamid/src/onesided/Makefile.mk @@ -36,7 +36,9 @@ lib_lib@MPILIBNAME@_la_SOURCES += \ src/mpid/pamid/src/onesided/mpid_win_lock_all.c \ src/mpid/pamid/src/onesided/mpid_win_pscw.c \ src/mpid/pamid/src/onesided/mpid_win_put.c \ + src/mpid/pamid/src/onesided/mpid_win_shared_query.c \ src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c \ + src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c \ src/mpid/pamid/src/onesided/mpid_win_flush.c \ src/mpid/pamid/src/onesided/mpid_win_allocate.c \ src/mpid/pamid/src/onesided/mpid_win_sync.c \ diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c new file mode 100644 index 0000000..97525f5 --- /dev/null +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -0,0 +1,409 @@ +/* begin_generated_IBM_copyright_prolog */ +/* */ +/* This is an automatically generated copyright prolog. */ +/* After initializing, DO NOT MODIFY OR MOVE */ +/* --------------------------------------------------------------- */ +/* Licensed Materials - Property of IBM */ +/* Blue Gene/Q 5765-PER 5765-PRP */ +/* */ +/* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */ +/* US Government Users Restricted Rights - */ +/* Use, duplication, or disclosure restricted */ +/* by GSA ADP Schedule Contract with IBM Corp. */ +/* */ +/* --------------------------------------------------------------- */ +/* */ +/* end_generated_IBM_copyright_prolog */ +/* (C)Copyright IBM Corp. 2007, 2011 */ +/** + * \file src/onesided/mpid_win_allocate_shared.c + * \brief + */ +#include "mpidi_onesided.h" +#include <sys/shm.h> +#include <sys/ipc.h> +#include <sys/stat.h> + +#undef FUNCNAME +#define FUNCNAME MPID_Win_allocate_shared +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) + +#define SHM_KEY_TAIL 0xfdbc /* If this value is changed, change */ + /* in PMD (mp_pmd.c) as well. */ + +extern int mpidi_dynamic_tasking; +#define MPIDI_PAGESIZE ((MPI_Aint)pageSize) +#define MPIDI_PAGESIZE_MASK (~(MPIDI_PAGESIZE-1)) +#define MPIDI_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_PAGESIZE_MASK)) & MPIDI_PAGESIZE_MASK) + + +int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { + int rank,comm_size; + int mpi_errno=PAMI_SUCCESS; + + + rank=comm_ptr->rank; + comm_size = comm_ptr->local_size; + rank = comm_ptr->rank; + + *onNode=0; + if (comm_ptr->intranode_table[rank] != -1) + *onNode=1; + if (*onNode== 0) { + MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_CONFLICT, + return mpi_errno, "**rmaconflict"); + } + + return mpi_errno; +} + +int CheckSpaceType(MPID_Win **win_ptr, MPID_Info *info,int *noncontig) { + int mpi_errno=MPI_SUCCESS; + /* Check if we are allowed to allocate space non-contiguously */ + if (info != NULL) { + int alloc_shared_nctg_flag = 0; + char alloc_shared_nctg_value[MPI_MAX_INFO_VAL+1]; + MPIR_Info_get_impl(info, "alloc_shared_noncontig", MPI_MAX_INFO_VAL, + alloc_shared_nctg_value, &alloc_shared_nctg_flag); + if ((alloc_shared_nctg_flag == 1)) { + if (!strncmp(alloc_shared_nctg_value, "true", strlen("true"))) + (*win_ptr)->mpid.info_args.alloc_shared_noncontig = 1; + if (!strncmp(alloc_shared_nctg_value, "false", strlen("false"))) + (*win_ptr)->mpid.info_args.alloc_shared_noncontig = 0; + } + } + *noncontig = (*win_ptr)->mpid.info_args.alloc_shared_noncontig; + return mpi_errno; +} + +int GetPageSize(void *addr, ulong *pageSize) +{ + pid_t pid; + FILE *fp; + char Line[201],A1[50],A2[50]; + char fileName[100]; + char A3[50],A4[50]; + int i=0; + char *t1,*t2; + int len; + #ifndef REDHAT + char a='-'; + char *search = &a; + #endif + void *beg, *end; + int found=0; + int ps; + int j,k; + + *pageSize = 0; + pid = getpid(); + sprintf(fileName,"/proc/%d/smaps",pid); + /* linux-2.6.29 or greater, KernelPageSize in /proc/pid/smaps includes 4K, 16 MB */ + TRACE_ERR("fileName = %s addr=%p\n",fileName,addr); + fp = fopen(fileName,"r"); + if (fp == NULL) { + TRACE_ERR("fileName = %s open failed errno=%d\n",fileName,errno); + return errno; + } + while(fgets(Line,200,fp)) { + i++; + sscanf(Line,"%s %s %s %s \n",A1,A2,A3,A4); + if (memcmp(A1,"KernelPageSize",14)==0) { + j=atoi(A2); + if ((A3[0]=='k') || (A3[0]=='K')) + k=1024; + else if ((A3[0]=='m') || (A3[0]=='M')) + k=1048576; + else if ((A3[0]=='g') || (A3[0]=='G')) + k=0x40000000; /* 1 GB */ + else { + printf("ERROR unrecognized unit A3=%s\n",A3); + break; + } + *pageSize = (ulong)(j * k); + TRACE_ERR(" addr=%p pageSize=%ld %s(%d)\n", addr,*pageSize,__FILE__,__LINE__); + break; + } + } + fclose(fp); + if (*pageSize == 0) { + ps = getpagesize(); + *pageSize = (ulong) ps; + TRACE_ERR("LinuxPageSize %p not in %s getpagesize=%ld\n", addr,fileName,*pageSize); + } + return 0; +} + +#define MPIDI_PAGESIZE ((MPI_Aint)pageSize) +#define MPIDI_PAGESIZE_MASK (~(MPIDI_PAGESIZE-1)) +#define MPIDI_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_PAGESIZE_MASK)) & MPIDI_PAGESIZE_MASK) + + + +int +MPID_getSharedSegment(MPI_Aint size, + int disp_unit, + MPID_Info * info, + MPID_Comm * comm_ptr, + void **base_ptr, + MPID_Win ** win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + void **base_pp = (void **) base_ptr; + int i, k, comm_size, rank; + int shm_id; + uint32_t shm_key; + int node_rank; + MPI_Aint *node_sizes; + void * base_addr; + MPI_Aint *tmp_buf; + int errflag = FALSE; + int noncontig = FALSE; + MPI_Aint pageSize, len,new_size; + char *cp; + MPID_Win *win; + MPIDI_Win_info *winfo; + int shm_flag = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR; + + win = *win_ptr; + comm_size = win->comm_ptr->local_size; + rank = win->comm_ptr->rank; + tmp_buf = MPIU_Malloc( 2*comm_size*sizeof(MPI_Aint)); + + mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + + GetPageSize((void *) win_ptr, &pageSize); + win->mpid.shm->segment_len = 0; + if (comm_size == 1) { + if (size > 0) { + if (noncontig) + new_size = MPIDI_ROUND_UP_PAGESIZE(size); + else + new_size = size; + *base_pp = MPIU_Malloc(new_size); + #ifndef MPIDI_NO_ASSERT + MPID_assert(*base_pp != NULL); + #else + MPIU_ERR_CHKANDJUMP((*base_pp == NULL), mpi_errno, MPI_ERR_BUFFER, "**bufnull"); + #endif + } else if (size == 0) { + *base_pp = NULL; + } else { + MPIU_ERR_CHKANDSTMT(size >=0 , mpi_errno, MPI_ERR_SIZE,return mpi_errno, "**rmasize"); + } + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + win->mpid.shm->segment_len = new_size; + win->mpid.info[rank].base_addr = *base_pp; + } else { + tmp_buf[rank] = (MPI_Aint) size; + mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, + tmp_buf, 1 * sizeof(MPI_Aint), MPI_BYTE, + (*win_ptr)->comm_ptr, &errflag); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + + /* calculate total number of bytes needed */ + for (i = 0; i < comm_size; ++i) { + len = tmp_buf[i]; + if (noncontig) + /* Round up to next page size */ + win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(len); + else + win->mpid.shm->segment_len += len; + } + /* get shared segment */ + + shm_key=-1; + if (rank == 0) { + #ifdef DYNAMIC_TASKING + /* generate an appropriate key */ + if (!mpidi_dynamic_tasking) { + cp = getenv("MP_I_PMD_PID"); + if (cp) { + shm_key = atoi(cp); + shm_key = shm_key & 0x07ffffff; + shm_key = shm_key | 0x80000000; + } else { + cp = getenv("MP_PARTITION"); + if (cp ) { + shm_key = atol(cp); + shm_key = (shm_key << 16) + SHM_KEY_TAIL; + } else { + TRACE_ERR("ERROR MP_PARTITION not set \n"); + } + } + } else { + cp = getenv("MP_I_KEY_RANGE"); + if (cp) { + sscanf(cp, "0x%x", &shm_key); + shm_key = shm_key | 0x80; + } else { + TRACE_ERR("ERROR MP_I_KEY_RANGE not set \n"); + } + } + #else + cp = getenv("MP_I_PMD_PID"); + if (cp) { + shm_key = atoi(cp); + shm_key = shm_key & 0x07ffffff; + shm_key = shm_key | 0x80000000; + } else { + cp = getenv("MP_PARTITION"); + if (cp ) { + shm_key = atol(cp); +#ifdef __PE__ + shm_key = (shm_key << 16) + SHMCC_KEY_TAIL; +#else + shm_key = (shm_key << 16); +#endif + } else { + TRACE_ERR("ERROR MP_PARTITION not set \n"); + } + } + #endif + MPID_assert(shm_key != -1); + shm_id = shmget(shm_key, win->mpid.shm->segment_len, shm_flag); + MPIU_ERR_CHKANDJUMP((shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + win->mpid.shm->base_addr = (void *) shmat(shm_id,0,0); + MPIU_ERR_CHKANDJUMP((base_addr == NULL), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); + win->mpid.shm->allocated = 1; + /* successfully created shm segment */ + mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, comm_ptr, &errflag); + } else { /* task other than task 0 */ + mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, comm_ptr, &errflag); + MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); + MPID_assert(shm_key != -1); + shm_id = shmget(shm_key, 0, 0); + if (shm_id != -1) { /* shm segment is available */ + win->mpid.shm->base_addr = (void *) shmat(shm_id,0,0); + win->mpid.shm->allocated = 1; + MPIU_ERR_CHKANDJUMP((base_addr == (void *) -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + } else { /* node leader failed, no need to try here */ + MPIU_ERR_CHKANDJUMP((shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + } + } + mpi_errno = MPIR_Barrier_impl(comm_ptr, &errflag); + /* compute the base addresses of each process within the shared memory segment */ + { + win->base = win->mpid.shm->base_addr; + winfo = &win->mpid.info[rank]; + winfo->win = win; + winfo->disp_unit = disp_unit; + win->mpid.info[0].base_addr = win->mpid.shm->base_addr; + char *cur_base = (*win_ptr)->mpid.shm->base_addr; + for (i = 1; i < comm_size; ++i) { + size = tmp_buf[i]; + if (size) { + if (noncontig) + /* Round up to next page size */ + win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size)); + else + win->mpid.info[i].base_addr = (void *) ((MPI_Aint) cur_base + size); + cur_base = win->mpid.info[i].base_addr; + } else { + win->mpid.info[i].base_addr = NULL; + } + } + } + *base_pp = win->mpid.info[rank].base_addr; + } + +fn_exit: + MPIU_Free(tmp_buf); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ +fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ + +} + +/** + * \brief MPI-PAMI glue for MPI_Win_allocate_shared function + * + * Create a window object. Allocates a MPID_Win object and initializes it, + * then allocates the collective info array, initalizes our entry, and + * performs an Allgather to distribute/collect the rest of the array entries. + * On each process, it allocates memory of at least size bytes that is shared + * among all processes in comm, and returns a pointer to the locally allocated + * segment in base_ptr that can be used for load/store accesses on the calling + * process. The locally allocated memory can be the target of load/store accessses + * by remote processes; the base pointers for other processes can be queried using + * the function 'MPI_Win_shared_query'. + * + * The call also returns a window object that can be used by all processes in comm + * to perform RMA operations. The size argument may be different at each process + * and size = 0 is valid. It is the user''s responsibility to ensure that the + * communicator comm represents a group of processes that can create a shared + * memory segment that can be accessed by all processes in the group. The + * allocated memory is contiguous across process ranks unless the info key + * alloc_shared_noncontig is specified. Contiguous across process ranks means that + * the first address in the memory segment of process i is consecutive with the + * last address in the memory segment of process i - 1. This may enable the user + * to calculate remote address offsets with local information only. + * + * Input Parameters: + * \param[in] size size of window in bytes (nonnegative integer) + * \param[in] disp_unit local unit size for displacements, in bytes (positive integer) + * \param[in] info info argument (handle)) + * \param[in] comm_ptr intra-Communicator (handle) + * \param[out] base_ptr address of local allocated window segment + * \param[out] win_ptr window object returned by the call (handle) + * \return MPI_SUCCESS, MPI_ERR_ARG, MPI_ERR_COMM, MPI_ERR_INFO. MPI_ERR_OTHER, + * MPI_ERR_SIZE + */ +int +MPID_Win_allocate_shared(MPI_Aint size, + int disp_unit, + MPID_Info * info, + MPID_Comm * comm_ptr, + void **base_ptr, + MPID_Win ** win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + void **baseP = base_ptr; + MPIDI_Win_info *winfo; + MPID_Win *win; + int rank, comm_size; + int onNode; + + + mpi_errno =MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_SHARED, MPI_WIN_UNIFIED); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + win = *win_ptr; + mpi_errno=CheckRankOnNode(comm_ptr,&onNode); + MPIU_ERR_CHKANDJUMP((onNode == 0), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + + rank = (*win_ptr)->comm_ptr->rank; + comm_size = (*win_ptr)->comm_ptr->local_size; + win->mpid.shm = MPIU_Malloc(sizeof(MPIDI_Win_shm_t)); + win->mpid.shm->allocated=0; + MPID_assert(win->mpid.shm != NULL); + MPID_getSharedSegment(size, disp_unit,info,comm_ptr,baseP, win_ptr); + + win->base = *baseP; + winfo = &win->mpid.info[rank]; + winfo->win = win; + winfo->disp_unit = disp_unit; + win->base = (void *) MPIU_PtrToAint(winfo->base_addr); + winfo->base_addr = win->base; + + mpi_errno = MPIDI_Win_allgather(*baseP,size,win_ptr); + if (mpi_errno != MPI_SUCCESS) + return mpi_errno; + + *(void**) base_ptr = (void *) win->base; + + mpi_errno = MPIR_Barrier_impl(comm_ptr, &mpi_errno); + +fn_exit: + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ +fn_fail: + MPIU_Free(win->mpid.shm); + goto fn_exit; + /* --END ERROR HANDLING-- */ + +} + diff --git a/src/mpid/pamid/src/onesided/mpid_win_free.c b/src/mpid/pamid/src/onesided/mpid_win_free.c index 5155ce8..2ba83d9 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_free.c +++ b/src/mpid/pamid/src/onesided/mpid_win_free.c @@ -21,6 +21,17 @@ */ #include "mpidi_onesided.h" +int MPIDI_SHM_Win_free(MPID_Win **win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + /* Free shared memory region */ + /* free shm_base_addrs that's only used for shared memory windows */ + if ((*win_ptr)->mpid.shm->allocated) + mpi_errno = shmdt((*win_ptr)->base); + MPIU_Free((*win_ptr)->mpid.shm); + (*win_ptr)->mpid.shm = NULL; + return mpi_errno; +} /** * \brief MPI-PAMI glue for MPI_Win_free function @@ -41,6 +52,7 @@ MPID_Win_free(MPID_Win **win_ptr) MPID_Win *win = *win_ptr; size_t rank = win->comm_ptr->rank; + int errflag = FALSE; if(win->mpid.sync.origin_epoch_type != win->mpid.sync.target_epoch_type || (win->mpid.sync.origin_epoch_type != MPID_EPOTYPE_NONE && @@ -48,9 +60,11 @@ MPID_Win_free(MPID_Win **win_ptr) MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_SYNC, return mpi_errno, "**rmasync"); } - mpi_errno = MPIR_Barrier_impl(win->comm_ptr, &mpi_errno); - if (mpi_errno != MPI_SUCCESS) - return mpi_errno; + mpi_errno = MPIR_Barrier_impl(win->comm_ptr, &errflag); + MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**mpi_bcast"); + + if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) + mpi_errno=MPIDI_SHM_Win_free(win_ptr); struct MPIDI_Win_info *winfo = &win->mpid.info[rank]; #ifdef USE_PAMI_RDMA @@ -68,9 +82,12 @@ MPID_Win_free(MPID_Win **win_ptr) MPID_assert(rc == PAMI_SUCCESS); } #endif - MPIU_Free(win->mpid.info); MPIU_Free(win->mpid.origin); + if (win->mpid.work.msgQ) + MPIU_Free(win->mpid.work.msgQ); + if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) + MPIU_Free(win->base); MPIR_Comm_release(win->comm_ptr, 0); diff --git a/src/mpid/pamid/src/onesided/mpid_win_pscw.c b/src/mpid/pamid/src/onesided/mpid_win_pscw.c index 1ee4b4a..ff22460 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_pscw.c +++ b/src/mpid/pamid/src/onesided/mpid_win_pscw.c @@ -36,7 +36,7 @@ MPIDI_WinPost_post(pami_context_t context, void * _info) { MPIDI_WinPSCW_info * info = (MPIDI_WinPSCW_info*)_info; - unsigned peer, index; + unsigned peer, index, pid,i; MPID_Group *group = info->win->mpid.sync.pw.group; MPID_assert(group != NULL); MPIDI_Win_control_t msg = { @@ -44,7 +44,13 @@ MPIDI_WinPost_post(pami_context_t context, }; for (index=0; index < group->size; ++index) { - peer = group->lrank_to_lpid[index].lrank; + pid = group->lrank_to_lpid[index].lpid; + for (i=0;i < ((info->win)->comm_ptr->local_size); i++) { + if ((info->win)->comm_ptr->local_group->lrank_to_lpid[i].lpid == pid) { + peer = ((info->win)->comm_ptr->local_group->lrank_to_lpid[i].lrank); + break; + } + } MPIDI_WinCtrlSend(context, &msg, peer, info->win); } @@ -67,7 +73,7 @@ MPIDI_WinComplete_post(pami_context_t context, void * _info) { MPIDI_WinPSCW_info * info = (MPIDI_WinPSCW_info*)_info; - unsigned peer, index; + unsigned peer, index,pid,i; MPID_Group *group = info->win->mpid.sync.sc.group; MPID_assert(group != NULL); MPIDI_Win_control_t msg = { @@ -75,8 +81,14 @@ MPIDI_WinComplete_post(pami_context_t context, }; for (index=0; index < group->size; ++index) { - peer = group->lrank_to_lpid[index].lrank; - MPIDI_WinCtrlSend(context, &msg, peer, info->win); + pid = group->lrank_to_lpid[index].lpid; + for (i=0;i < ((info->win)->comm_ptr->local_size); i++) { + if ((info->win)->comm_ptr->local_group->lrank_to_lpid[i].lpid == pid) { + peer = ((info->win)->comm_ptr->local_group->lrank_to_lpid[i].lrank); + break; + } + } + MPIDI_WinCtrlSend(context, &msg, peer, info->win); } info->done = 1; diff --git a/src/mpid/pamid/src/onesided/mpid_win_shared_query.c b/src/mpid/pamid/src/onesided/mpid_win_shared_query.c new file mode 100644 index 0000000..fb304ee --- /dev/null +++ b/src/mpid/pamid/src/onesided/mpid_win_shared_query.c @@ -0,0 +1,50 @@ +/* begin_generated_IBM_copyright_prolog */ +/* */ +/* This is an automatically generated copyright prolog. */ +/* After initializing, DO NOT MODIFY OR MOVE */ +/* --------------------------------------------------------------- */ +/* Licensed Materials - Property of IBM */ +/* Blue Gene/Q 5765-PER 5765-PRP */ +/* */ +/* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */ +/* US Government Users Restricted Rights - */ +/* Use, duplication, or disclosure restricted */ +/* by GSA ADP Schedule Contract with IBM Corp. */ +/* */ +/* --------------------------------------------------------------- */ +/* */ +/* end_generated_IBM_copyright_prolog */ +/* (C)Copyright IBM Corp. 2007, 2011 */ +/** + * \file src/onesided/mpid_win_shared_query.c + * \brief queries the process-local address for remote memory segments + * created with MPI_Win_allocate_shared. + */ +#include "mpidi_onesided.h" + +/** + * \brief MPI-PAMI glue for MPI_Win_shared_query function + * + * Query the size and base pointer for a patch of a shared memory window + * + * \param[in] win shared memory window object + * \param[in] rank rank in the group of window win or MPI_PROC_NULL + * \param[out] size size of the window segment (non-negative integer) + * \param[out] disp_unit local unit size for displacements, in bytes + * \param[out] base_ptr address for load/store access to window segment + * \return MPI_SUCCESS, MPI_ERR_OTHER, or error returned from + */ + +int +MPID_Win_shared_query(MPID_Win *win, int rank, MPI_Aint *size, + int *disp_unit, void *base_ptr) +{ + int mpi_errno = MPI_SUCCESS; + + *(void**) base_ptr = (void *) win->base; + *size = win->size; + *disp_unit = win->disp_unit; + + return mpi_errno; +} + ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_datatypes.h | 27 +- .../pamid/src/dyntask/mpid_comm_spawn_multiple.c | 5 +- src/mpid/pamid/src/dyntask/mpidi_port.c | 1 - src/mpid/pamid/src/misc/mpid_unimpl.c | 14 - src/mpid/pamid/src/mpid_finalize.c | 7 + src/mpid/pamid/src/mpidi_util.c | 2 +- src/mpid/pamid/src/onesided/Makefile.mk | 2 + src/mpid/pamid/src/onesided/mpid_1s.c | 10 +- src/mpid/pamid/src/onesided/mpid_win_accumulate.c | 1 - src/mpid/pamid/src/onesided/mpid_win_allocate.c | 2 +- .../pamid/src/onesided/mpid_win_allocate_shared.c | 439 ++++++++++++++++++++ .../pamid/src/onesided/mpid_win_compare_and_swap.c | 6 - src/mpid/pamid/src/onesided/mpid_win_create.c | 5 +- .../pamid/src/onesided/mpid_win_create_dynamic.c | 2 +- .../pamid/src/onesided/mpid_win_fetch_and_op.c | 6 - src/mpid/pamid/src/onesided/mpid_win_flush.c | 52 ++-- src/mpid/pamid/src/onesided/mpid_win_free.c | 43 ++- src/mpid/pamid/src/onesided/mpid_win_get.c | 2 - .../pamid/src/onesided/mpid_win_get_accumulate.c | 135 +++---- src/mpid/pamid/src/onesided/mpid_win_lock.c | 11 +- src/mpid/pamid/src/onesided/mpid_win_lock_all.c | 91 +++- src/mpid/pamid/src/onesided/mpid_win_pscw.c | 22 +- src/mpid/pamid/src/onesided/mpid_win_put.c | 2 - .../{mpid_win_attach.c => mpid_win_shared_query.c} | 46 +-- src/mpid/pamid/src/onesided/mpidi_onesided.h | 70 +++- src/mpid/pamid/src/onesided/mpidi_win_control.c | 5 + 26 files changed, 783 insertions(+), 225 deletions(-) create mode 100644 src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c copy src/mpid/pamid/src/onesided/{mpid_win_attach.c => mpid_win_shared_query.c} (56%) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-141-g399e546
by noreply＠mpich.org 08 Jan '14

08 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 399e546369c3938ada9cc0b6c3c218f685c7613b (commit) from 02241e287c99d238a5407c2e186b00cb1b71ca2f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/399e546369c3938ada9cc0b6c3c218f68… commit 399e546369c3938ada9cc0b6c3c218f685c7613b Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Tue Jan 7 23:32:29 2014 +0000 pamid is capable of supporting WIN_UNIFIED. Renamed WIN_SEPARATE to WIN_UNIFIED. pamid's WIN_SYNC already does a full memory barrier, so we should be all set for UNIFIED support. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate.c b/src/mpid/pamid/src/onesided/mpid_win_allocate.c index 227d75d..d17ff26 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate.c @@ -61,7 +61,7 @@ MPID_Win_allocate(MPI_Aint size, MPID_Win *win; int rank; - rc=MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_ALLOCATE, MPI_WIN_SEPARATE); + rc=MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_ALLOCATE, MPI_WIN_UNIFIED); win = *win_ptr; if (size > 0) { diff --git a/src/mpid/pamid/src/onesided/mpid_win_create.c b/src/mpid/pamid/src/onesided/mpid_win_create.c index 7becc3e..38f072a 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_create.c +++ b/src/mpid/pamid/src/onesided/mpid_win_create.c @@ -199,7 +199,7 @@ MPID_Win_create(void * base, pami_task_t taskid; MPIDI_Win_info *winfo; - rc=MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_CREATE, MPI_WIN_SEPARATE); + rc=MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_CREATE, MPI_WIN_UNIFIED); win = *win_ptr; win->base = base; rank = comm_ptr->rank; diff --git a/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c b/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c index 1f30513..5343362 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c +++ b/src/mpid/pamid/src/onesided/mpid_win_create_dynamic.c @@ -49,7 +49,7 @@ MPID_Win_create_dynamic( MPID_Info * info, MPID_Win *win; int rank,i; - rc=MPIDI_Win_init(0,1,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_DYNAMIC, MPI_WIN_SEPARATE); + rc=MPIDI_Win_init(0,1,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_DYNAMIC, MPI_WIN_UNIFIED); win = *win_ptr; win->base = MPI_BOTTOM; ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/src/onesided/mpid_win_allocate.c | 2 +- src/mpid/pamid/src/onesided/mpid_win_create.c | 2 +- .../pamid/src/onesided/mpid_win_create_dynamic.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-140-g02241e2
by noreply＠mpich.org 07 Jan '14

07 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 02241e287c99d238a5407c2e186b00cb1b71ca2f (commit) from 4f97779716a487b66b301e16c2cb9b75dccdff8e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/02241e287c99d238a5407c2e186b00cb1… commit 02241e287c99d238a5407c2e186b00cb1b71ca2f Author: Rob Latham <robl(a)mcs.anl.gov> Date: Tue Jan 7 10:06:17 2014 -0600 Use ROMIO pointer-casting routines the code in glue_romio replicates a bit of pointer-size assertion logic. it's only used by ROMIO, though. if mpitypedefs.h is not included (see recent standalone-romio cleanups) we will get an undefined symbol at link time (MPI_AINT_CAST_TO_VOID_PTR). Use equivalent ADIOI_AINT_CAST_TO_VOID_PTR instead for this ROMIO-related code. Signed-off-by: Jed Brown <jedbrown(a)mcs.anl.gov> diff --git a/src/include/glue_romio.h.in b/src/include/glue_romio.h.in index 4d60ed2..f074353 100644 --- a/src/include/glue_romio.h.in +++ b/src/include/glue_romio.h.in @@ -42,9 +42,10 @@ extern int MPIR_Ext_dbg_romio_terse_enabled; extern int MPIR_Ext_dbg_romio_typical_enabled; extern int MPIR_Ext_dbg_romio_verbose_enabled; -/* a copy of MPID_Ensure_Aint_fits_in_pointer for external use */ +/* a copy of MPID_Ensure_Aint_fits_in_pointer for external use, slightly + * modified to use ROMIO's version of the pointer-casting macro */ #define MPIR_Ext_ensure_Aint_fits_in_pointer(aint) \ - MPIR_Ext_assert((aint) == (MPI_Aint)(MPIR_Upint) MPI_AINT_CAST_TO_VOID_PTR(aint)); + MPIR_Ext_assert((aint) == (MPI_Aint)(MPIR_Upint) ADIOI_AINT_CAST_TO_VOID_PTR(aint)); /* to be called early by ROMIO's initialization process in order to setup init-time * glue code that cannot be initialized statically */ ----------------------------------------------------------------------- Summary of changes: src/include/glue_romio.h.in | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-139-g4f97779
by noreply＠mpich.org 07 Jan '14

07 Jan '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 4f97779716a487b66b301e16c2cb9b75dccdff8e (commit) via 0743c396ecfc07cef7769970a4f02edc612f5fba (commit) from 57e54ea330798e1b2817eccb683ad4476b03fcfc (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/4f97779716a487b66b301e16c2cb9b75d… commit 4f97779716a487b66b301e16c2cb9b75dccdff8e Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Fri Jan 3 12:52:04 2014 -0600 Make SMP-awareness in MPI_Barrier consistent. The current implementation of SMP-awareness in MPI_Barrier was in MPIR_Barrier_impl. This makes the default implementation of barrier SMP-aware. However, if a device overrides barrier and then calls back the default implementation through MPIR_Barrier, it can no longer take advantage of SMP-awareness. This patch moves the SMP-aware implementation to MPIR_Barrier_intra, which is called by MPIR_Barrier_impl through MPIR_Barrier, in the default implementation. See #1957. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpi/coll/barrier.c b/src/mpi/coll/barrier.c index d81e201..2583a75 100644 --- a/src/mpi/coll/barrier.c +++ b/src/mpi/coll/barrier.c @@ -33,8 +33,6 @@ cvars: #endif /* -- End Profiling Symbol Block */ -PMPI_LOCAL int MPIR_Barrier_or_coll_fn(MPID_Comm *comm_ptr, int *errflag ); - /* Define MPICH_MPI_FROM_PMPI if weak symbols are not supported to build the MPI routines */ #ifndef MPICH_MPI_FROM_PMPI @@ -63,6 +61,66 @@ PMPI_LOCAL int MPIR_Barrier_or_coll_fn(MPID_Comm *comm_ptr, int *errflag ); This is an intracommunicator barrier only! */ +#undef FUNCNAME +#define FUNCNAME barrier_smp_intra +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +static int barrier_smp_intra(MPID_Comm *comm_ptr, int *errflag) +{ + int mpi_errno=MPI_SUCCESS; + int mpi_errno_ret = MPI_SUCCESS; + + MPIU_Assert(MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_BARRIER && + MPIR_Comm_is_node_aware(comm_ptr)); + + /* do the intranode barrier on all nodes */ + if (comm_ptr->node_comm != NULL) + { + mpi_errno = MPIR_Barrier_impl(comm_ptr->node_comm, errflag); + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = TRUE; + MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); + MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); + } + } + + /* do the barrier across roots of all nodes */ + if (comm_ptr->node_roots_comm != NULL) { + mpi_errno = MPIR_Barrier_impl(comm_ptr->node_roots_comm, errflag); + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = TRUE; + MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); + MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); + } + } + + /* release the local processes on each node with a 1-byte + broadcast (0-byte broadcast just returns without doing + anything) */ + if (comm_ptr->node_comm != NULL) + { + int i=0; + mpi_errno = MPIR_Bcast_impl(&i, 1, MPI_BYTE, 0, comm_ptr->node_comm, errflag); + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = TRUE; + MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); + MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); + } + } + + fn_exit: + if (mpi_errno_ret) + mpi_errno = mpi_errno_ret; + else if (*errflag) + MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**coll_fail"); + return mpi_errno; + fn_fail: + goto fn_exit; +} + /* not declared static because it is called in ch3_comm_connect/accept */ #undef FUNCNAME #define FUNCNAME MPIR_Barrier_intra @@ -82,6 +140,18 @@ int MPIR_Barrier_intra( MPID_Comm *comm_ptr, int *errflag ) /* Trivial barriers return immediately */ if (size == 1) goto fn_exit; + if (MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_BARRIER && + MPIR_Comm_is_node_aware(comm_ptr)) { + mpi_errno = barrier_smp_intra(comm_ptr, errflag); + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = TRUE; + MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); + MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); + } + goto fn_exit; + } + rank = comm_ptr->rank; comm = comm_ptr->handle; @@ -113,37 +183,6 @@ int MPIR_Barrier_intra( MPID_Comm *comm_ptr, int *errflag ) goto fn_exit; } -/* A simple utility function to that calls the comm_ptr->coll_fns->Barrier -override if it exists or else it calls MPIR_Barrier_intra with the same arguments. */ -/* Note that this function must *not* be inline - if weak symbols are not - available, this function must be a global symbol. */ -#undef FUNCNAME -#define FUNCNAME MPIR_Barrier_or_coll_fn -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -PMPI_LOCAL int MPIR_Barrier_or_coll_fn(MPID_Comm *comm_ptr, int *errflag ) -{ - int mpi_errno = MPI_SUCCESS; - - if (comm_ptr->coll_fns != NULL && comm_ptr->coll_fns->Barrier != NULL) - { - /* --BEGIN USEREXTENSION-- */ - mpi_errno = comm_ptr->coll_fns->Barrier(comm_ptr, errflag); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - /* --END USEREXTENSION-- */ - } - else { - mpi_errno = MPIR_Barrier_intra(comm_ptr, errflag); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - - /* not declared static because a machine-specific function may call this one in some cases */ #undef FUNCNAME @@ -285,57 +324,8 @@ int MPIR_Barrier_impl(MPID_Comm *comm_ptr, int *errflag) } else { - if (comm_ptr->comm_kind == MPID_INTRACOMM) { - if (MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_BARRIER && - MPIR_Comm_is_node_aware(comm_ptr)) { - - /* do the intranode barrier on all nodes */ - if (comm_ptr->node_comm != NULL) - { - mpi_errno = MPIR_Barrier_or_coll_fn(comm_ptr->node_comm, errflag); - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = TRUE; - MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); - MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); - } - } - - /* do the barrier across roots of all nodes */ - if (comm_ptr->node_roots_comm != NULL) { - mpi_errno = MPIR_Barrier_or_coll_fn(comm_ptr->node_roots_comm, errflag); - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = TRUE; - MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); - MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); - } - } - - /* release the local processes on each node with a 1-byte broadcast - (0-byte broadcast just returns without doing anything) */ - if (comm_ptr->node_comm != NULL) - { - int i=0; - mpi_errno = MPIR_Bcast_impl(&i, 1, MPI_BYTE, 0, comm_ptr->node_comm, errflag); - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = TRUE; - MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail"); - MPIU_ERR_ADD(mpi_errno_ret, mpi_errno); - } - } - } - else { - mpi_errno = MPIR_Barrier_intra( comm_ptr, errflag ); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } - } - else { - /* intercommunicator */ - mpi_errno = MPIR_Barrier_inter( comm_ptr, errflag ); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } + mpi_errno = MPIR_Barrier(comm_ptr, errflag); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); } fn_exit: http://git.mpich.org/mpich.git/commitdiff/0743c396ecfc07cef7769970a4f02edc6… commit 0743c396ecfc07cef7769970a4f02edc612f5fba Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Fri Jan 3 12:37:32 2014 -0600 Remove duplicate code. Use MPIR_Bcast as an inline function, and let the compiler do its job, instead of duplicating the code. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpi/coll/bcast.c b/src/mpi/coll/bcast.c index 5860876..ce3be85 100644 --- a/src/mpi/coll/bcast.c +++ b/src/mpi/coll/bcast.c @@ -1454,19 +1454,8 @@ int MPIR_Bcast_impl(void *buffer, int count, MPI_Datatype datatype, int root, MP } else { - if (comm_ptr->comm_kind == MPID_INTRACOMM) - { - /* intracommunicator */ - mpi_errno = MPIR_Bcast_intra( buffer, count, datatype, root, comm_ptr, errflag ); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - - } - else - { - /* intercommunicator */ - mpi_errno = MPIR_Bcast_inter( buffer, count, datatype, root, comm_ptr, errflag ); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } + mpi_errno = MPIR_Bcast(buffer, count, datatype, root, comm_ptr, errflag); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); } @@ -1483,7 +1472,7 @@ int MPIR_Bcast_impl(void *buffer, int count, MPI_Datatype datatype, int root, MP #define FUNCNAME MPIR_Bcast #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -int MPIR_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPID_Comm *comm_ptr, int *errflag) +inline int MPIR_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPID_Comm *comm_ptr, int *errflag) { int mpi_errno = MPI_SUCCESS; ----------------------------------------------------------------------- Summary of changes: src/mpi/coll/barrier.c | 158 ++++++++++++++++++++++------------------------- src/mpi/coll/bcast.c | 17 +---- 2 files changed, 77 insertions(+), 98 deletions(-) hooks/post-receive -- MPICH primary repository

1 0