This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 58d9076d12a9f5b4a15965dd4250b1a6a1ddf00b (commit) via e38c59a82a0805d60e40c577aa09436e70295027 (commit) via 4dbe135a9ddedb5ad72a4abd0aafb75843140763 (commit) from b9d677473725748de832f7a67d5835baddfa6279 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/58d9076d12a9f5b4a15965dd4250b1a6a1... commit 58d9076d12a9f5b4a15965dd4250b1a6a1ddf00b Author: Sameer Kumar <[email protected]> Date: Tue Apr 8 09:20:35 2014 -0500 pamid: fixes for MPI_Win_allocate_shared on BG/Q. Tested using the mpich testsuite and the following command: make testing MPITEST_PROGRAM_WRAPPER=" --ranks-per-node=8 --block R00-M1-N10 --timeout 60 --envs BG_MAPCOMMONHEAP=1 : " MPIEXEC=runjob V=1 diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 7d70b87..332cf47 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -48,15 +48,28 @@ int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { *onNode=1; - if (comm_ptr->intranode_table == NULL) - *onNode = 0; - else +#ifdef __PE__ for (i=0; i< comm_size; i++) { if (comm_ptr->intranode_table[i] == -1) { *onNode=0; break; } } +#else +#ifdef PAMIX_IS_LOCAL_TASK + for (i=0; i< comm_size; i++) { + if (!PAMIX_Task_is_local(comm_ptr->vcr[i]->taskid)) { + *onNode=0; + break; + } + } +#else + if (comm_ptr->intranode_table == NULL) + *onNode = 0; +#endif +#endif + + if (*onNode== 0) { MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_CONFLICT, return mpi_errno, "**rmaconflict"); @@ -374,32 +387,71 @@ MPID_Win_allocate_shared(MPI_Aint size, MPID_Win ** win_ptr) { int mpi_errno = MPI_SUCCESS; + int onNode = 0; + MPID_Win *win = NULL; + int rank; + void **baseP = base_ptr; MPIDI_Win_info *winfo; - MPID_Win *win; - int rank, comm_size,i; - int onNode,noncontig=FALSE; + int comm_size,i; + int noncontig=FALSE; MPI_Aint pageSize=0; - - + + /* Verify all ranks are on-node */ + mpi_errno=CheckRankOnNode(comm_ptr,&onNode); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPIU_ERR_CHKANDJUMP((onNode == 0), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + /* Initialize the window */ mpi_errno =MPIDI_Win_init(size,disp_unit,win_ptr, info, comm_ptr, MPI_WIN_FLAVOR_SHARED, MPI_WIN_UNIFIED); if (mpi_errno) MPIU_ERR_POP(mpi_errno); win = *win_ptr; - mpi_errno=CheckRankOnNode(comm_ptr,&onNode); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - MPIU_ERR_CHKANDJUMP((onNode == 0), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); - rank = (*win_ptr)->comm_ptr->rank; - comm_size = (*win_ptr)->comm_ptr->local_size; win->mpid.shm = MPIU_Malloc(sizeof(MPIDI_Win_shm_t)); - win->mpid.shm->allocated=0; MPID_assert(win->mpid.shm != NULL); + memset(win->mpid.shm, 0, sizeof(MPIDI_Win_shm_t)); + + rank = comm_ptr->rank; + win->mpid.info[rank].win = win; + win->mpid.info[rank].disp_unit = disp_unit; + +#ifdef __BGQ__ + /* verify BG_MAPCOMMONHEAP=1 env. variable is set */ + if (rank == 0) { + assert(NULL!=getenv("BG_MAPCOMMONHEAP")); + baseP = MPIU_Malloc(size+sizeof(pthread_mutex_t)); +#ifdef MPIDI_NO_ASSERT + MPIU_ERR_CHKANDJUMP((baseP == NULL), mpi_errno, MPI_ERR_BUFFER, "**bufnull"); +#else + MPID_assert(baseP != NULL); +#endif + + pthread_mutex_t *mutex = (pthread_mutex_t *)(((uintptr_t) baseP) + size); + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutex_init(mutex, &attr); + } + + int errflag = 0; + mpi_errno = MPIR_Bcast_impl(&baseP, sizeof(char*), MPI_BYTE, 0, + win->comm_ptr, &errflag); + + win->mpid.shm->mutex_lock = (pthread_mutex_t *)(((uintptr_t) baseP) + size); + win->mpid.shm->allocated = 1; + win->mpid.shm->base_addr = baseP; + + win->base = baseP; + win->mpid.info[rank].base_addr = baseP; + + mpi_errno = MPIDI_Win_allgather(size,win_ptr); + if (mpi_errno != MPI_SUCCESS) { + MPIU_Free(win->mpid.shm); + return mpi_errno; + } +#else + mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); + comm_size = (*win_ptr)->comm_ptr->local_size; MPID_getSharedSegment(size, disp_unit,comm_ptr,baseP, win_ptr,&pageSize,&noncontig); - winfo = &win->mpid.info[rank]; - winfo->win = win; - winfo->disp_unit = disp_unit; mpi_errno = MPIDI_Win_allgather(size,win_ptr); if (mpi_errno != MPI_SUCCESS) return mpi_errno; @@ -419,15 +471,18 @@ MPID_Win_allocate_shared(MPI_Aint size, } } } +#endif + *(void**) base_ptr = (void *) win->mpid.info[rank].base_addr; mpi_errno = MPIR_Barrier_impl(comm_ptr, &mpi_errno); - fn_exit: return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: - MPIU_Free(win->mpid.shm); + if (win != NULL) + if (win->mpid.shm != NULL) + MPIU_Free(win->mpid.shm); goto fn_exit; /* --END ERROR HANDLING-- */ diff --git a/src/mpid/pamid/src/onesided/mpid_win_free.c b/src/mpid/pamid/src/onesided/mpid_win_free.c index a9399ac..1c5a7ec 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_free.c +++ b/src/mpid/pamid/src/onesided/mpid_win_free.c @@ -31,24 +31,35 @@ int MPIDI_SHM_Win_free(MPID_Win **win_ptr) int rc; int mpi_errno = MPI_SUCCESS; - /* Free shared memory region */ - /* free shm_base_addrs that's only used for shared memory windows */ - if ((*win_ptr)->mpid.shm->allocated) { - OPA_fetch_and_add_int((OPA_int_t *) (*win_ptr)->mpid.shm->shm_count,-1); - while(*(*win_ptr)->mpid.shm->shm_count) MPIDI_QUICKSLEEP; - if ((*win_ptr)->comm_ptr->rank == 0) { - MPIDI_SHM_MUTEX_DESTROY(*win_ptr); - } - mpi_errno = shmdt((*win_ptr)->mpid.shm->base_addr); - if ((*win_ptr)->comm_ptr->rank == 0) { - rc=shmctl((*win_ptr)->mpid.shm->shm_id,IPC_RMID,NULL); - MPIU_ERR_CHKANDJUMP((rc == -1), errno,MPI_ERR_RMA_SHARED, "**shmctl"); - } +#ifdef __BGQ__ + if ((*win_ptr)->comm_ptr->rank == 0) { + MPIDI_SHM_MUTEX_DESTROY(*win_ptr); + MPIU_Free ((*win_ptr)->base); + } + (*win_ptr)->mpid.shm = NULL; +#endif + +#ifdef __PE__ + /* Free shared memory region */ + /* free shm_base_addrs that's only used for shared memory windows */ + if ((*win_ptr)->mpid.shm->allocated) { + OPA_fetch_and_add_int((OPA_int_t *) (*win_ptr)->mpid.shm->shm_count,-1); + while(*(*win_ptr)->mpid.shm->shm_count) MPIDI_QUICKSLEEP; + if ((*win_ptr)->comm_ptr->rank == 0) { + MPIDI_SHM_MUTEX_DESTROY(*win_ptr); + } + mpi_errno = shmdt((*win_ptr)->mpid.shm->base_addr); + if ((*win_ptr)->comm_ptr->rank == 0) { + rc=shmctl((*win_ptr)->mpid.shm->shm_id,IPC_RMID,NULL); + MPIU_ERR_CHKANDJUMP((rc == -1), errno,MPI_ERR_RMA_SHARED, "**shmctl"); } - MPIU_Free((*win_ptr)->mpid.shm); - (*win_ptr)->mpid.shm = NULL; - fn_fail: - return mpi_errno; + } + MPIU_Free((*win_ptr)->mpid.shm); + (*win_ptr)->mpid.shm = NULL; +#endif + + fn_fail: + return mpi_errno; } /** diff --git a/src/mpid/pamid/src/pamix/pamix.c b/src/mpid/pamid/src/pamix/pamix.c index 4a49c9f..1d399b1 100644 --- a/src/mpid/pamid/src/pamix/pamix.c +++ b/src/mpid/pamid/src/pamix/pamix.c @@ -23,6 +23,7 @@ #include <assert.h> #include <limits.h> +#include "mpidimpl.h" #include <pamix.h> #define PAMIX_assert_always(x) assert(x) http://git.mpich.org/mpich.git/commitdiff/e38c59a82a0805d60e40c577aa09436e70... commit e38c59a82a0805d60e40c577aa09436e70295027 Author: Michael Blocksome <[email protected]> Date: Tue Apr 8 16:44:13 2014 -0500 pamid: support downlevel pami 'is_local_task' extension diff --git a/src/mpid/pamid/include/mpidi_platform.h b/src/mpid/pamid/include/mpidi_platform.h index 4a280aa..ce6b232 100644 --- a/src/mpid/pamid/include/mpidi_platform.h +++ b/src/mpid/pamid/include/mpidi_platform.h @@ -24,6 +24,10 @@ #ifndef __include_mpidi_platform_h__ #define __include_mpidi_platform_h__ +/* Allow MPICH to detect local tasks */ +#define MPID_USE_NODE_IDS +typedef int32_t MPID_Node_id_t; + /* Default values */ #define MPIDI_MAX_CONTEXTS 64 @@ -140,10 +144,6 @@ static const char _ibm_release_version_[] = "V1R2M0"; #define DYNAMIC_TASKING 1 #define QUEUE_BINARY_SEARCH_SUPPORT 1 -/* Allow MPICH to detect local tasks */ -#define MPID_USE_NODE_IDS 1 -typedef int32_t MPID_Node_id_t; - /* 'is local task' extension and limits */ #define PAMIX_IS_LOCAL_TASK #define PAMIX_IS_LOCAL_TASK_STRIDE (1) diff --git a/src/mpid/pamid/include/pamix.h b/src/mpid/pamid/include/pamix.h index 6adbab5..80ef77a 100644 --- a/src/mpid/pamid/include/pamix.h +++ b/src/mpid/pamid/include/pamix.h @@ -102,6 +102,11 @@ void PAMIX_Progress_disable(pami_context_t context, pamix_progress_t event_type); +pami_result_t +PAMIX_is_local_task_get_node_info(pami_task_t task, + uint32_t *node_id, + uint32_t *offset, + uint32_t *max_nodes); #ifdef __BGQ__ diff --git a/src/mpid/pamid/src/pamix/pamix.c b/src/mpid/pamid/src/pamix/pamix.c index ab16295..4a49c9f 100644 --- a/src/mpid/pamid/src/pamix/pamix.c +++ b/src/mpid/pamid/src/pamix/pamix.c @@ -36,6 +36,7 @@ #ifdef __BGQ__ #define __BG__ +#include <spi/include/kernel/location.h> #endif typedef pami_result_t (*pamix_progress_register_fn) (pami_context_t context, @@ -81,7 +82,6 @@ pamix_extension_info_t PAMIX_Extensions; ({ \ void* fn; \ fn = PAMI_Extension_symbol(ext, name); \ - PAMIX_assert_always(fn != NULL); \ (type)fn; \ }) void @@ -115,6 +115,13 @@ PAMIX_Initialize(pami_client_t client) PAMIX_Extensions.is_local_task.bitmask = PAMI_EXTENSION_FUNCTION(uintptr_t, "bitmask", PAMIX_Extensions.is_local_task.extension); #if defined(MPID_USE_NODE_IDS) PAMIX_Extensions.is_local_task.node_info = PAMI_EXTENSION_FUNCTION(node_info_fn, "get_node_info", PAMIX_Extensions.is_local_task.extension); + if (PAMIX_Extensions.is_local_task.node_info == NULL) + { + /* The "node information" is not available via the "is_local_task" + * extension to pami - possibly due to a downlevel version of pami. + * */ + PAMIX_Extensions.is_local_task.node_info = (node_info_fn) PAMIX_is_local_task_get_node_info; + } #endif } @@ -268,6 +275,36 @@ PAMIX_Progress_disable(pami_context_t context, } +pami_result_t +PAMIX_is_local_task_get_node_info(pami_task_t task, + uint32_t *node_id, + uint32_t *offset, + uint32_t *max_nodes) +{ +#if defined(__BG__) + size_t coords[5]; + if (PAMI_SUCCESS == PAMIX_Task2torus(task, coords)) + { + const pamix_torus_info_t * info = PAMIX_Torus_info(); + + *node_id = + coords[0] + + coords[1] * info->size[0] + + coords[2] * info->size[0] * info->size[1] + + coords[3] * info->size[0] * info->size[1] * info->size[2] + + coords[1] * info->size[0] * info->size[1] * info->size[2] * info->size[3]; + + *max_nodes = info->size[0] * info->size[1] * info->size[2] * info->size[3] * info->size[4]; + + *offset = 0; /* ???????????????????????????????? */ + } +#else + MPID_abort(); /* Other platforms should not need a fallback implementation */ +#endif + + return PAMI_SUCCESS; +} + #if defined(__BG__) const pamix_torus_info_t * http://git.mpich.org/mpich.git/commitdiff/4dbe135a9ddedb5ad72a4abd0aafb75843... commit 4dbe135a9ddedb5ad72a4abd0aafb75843140763 Author: Michael Blocksome <[email protected]> Date: Thu Apr 17 15:47:58 2014 -0500 pamid: mpid_win_* fixes diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index dea20d7..7d70b87 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -47,10 +47,14 @@ int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { comm_size = comm_ptr->local_size; *onNode=1; - for (i=0; i< comm_size; i++) { + + if (comm_ptr->intranode_table == NULL) + *onNode = 0; + else + for (i=0; i< comm_size; i++) { if (comm_ptr->intranode_table[i] == -1) { - *onNode=0; - break; + *onNode=0; + break; } } if (*onNode== 0) { diff --git a/src/mpid/pamid/src/onesided/mpid_win_free.c b/src/mpid/pamid/src/onesided/mpid_win_free.c index c8a8867..a9399ac 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_free.c +++ b/src/mpid/pamid/src/onesided/mpid_win_free.c @@ -86,6 +86,9 @@ MPID_Win_free(MPID_Win **win_ptr) + if (win->create_flavor == MPI_WIN_FLAVOR_ALLOCATE) + MPIU_Free(win->base); + struct MPIDI_Win_info *winfo = &win->mpid.info[rank]; #ifdef USE_PAMI_RDMA if (win->size != 0) ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_platform.h | 8 +- src/mpid/pamid/include/pamix.h | 5 + .../pamid/src/onesided/mpid_win_allocate_shared.c | 99 ++++++++++++++++---- src/mpid/pamid/src/onesided/mpid_win_free.c | 48 ++++++---- src/mpid/pamid/src/pamix/pamix.c | 40 ++++++++- 5 files changed, 158 insertions(+), 42 deletions(-) hooks/post-receive -- MPICH primary repository