This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3e6b4e15c386b8bc6f2e7fea20f1cfc15c55487c (commit) via f31656cbb6dbfe63c985150bb4a7bd21e99b14bd (commit) via e08bc140542eda7384e7172046312c11010cb561 (commit) via c58cec82601da186ee944d5151df4f5c6e734212 (commit) via a6dae684eacd22d43e275e47e015a8b00054bf7e (commit) via 6a928e7ec1f4c27190959e0e780aec202cad4cd9 (commit) via faa3a51fd280dac9df6ee538c4510dfe06dc0374 (commit) via 8bb4b63034407d8f8e85193617110b5842e25e6f (commit) via 20ff85b8c45d03c0b3f42dda13ed718bf64ab3fb (commit) via 3e23947322ee6257e66eb7d316de01ffbda3ec3c (commit) via 0a338e5dbe6a3589a9da8ba467d4bc18d3f94270 (commit) via 48aaa82295c800f976c49a7aaba4344175d643b5 (commit) via 723e5d5e86d33e5c668dfc3b528618a88587b928 (commit) via 8bf8f815a5105af42fb368f5da4b71854620d90f (commit) via c7bc4694fbb0b10f6a09141794c6e0664a76197a (commit) via a64a1d2ce36c1b0bbf4779734a67aa6643192f30 (commit) via 66221968523896310b3920cbfd44f4c1fd9bc205 (commit) from 08273c868fc7ddd7d81473e153eb3a0f7d38fabf (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3e6b4e15c386b8bc6f2e7fea20f1cfc15c... commit 3e6b4e15c386b8bc6f2e7fea20f1cfc15c55487c Author: Pavan Balaji <[email protected]> Date: Thu Aug 1 17:56:26 2013 -0500 Warning squash. diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index e469edc..17162a7 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -533,7 +533,7 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ void *base = NULL; MPI_User_function *uop = NULL; MPID_Datatype *dtp; - int origin_predefined, result_predefined, target_predefined; + int origin_predefined, target_predefined; MPIDI_VC_t *orig_vc, *target_vc; int mpi_errno = MPI_SUCCESS; MPIU_CHKLMEM_DECL(2); @@ -545,7 +545,6 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ if (op != MPI_NO_OP) { MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, origin_predefined); } - MPIDI_CH3I_DATATYPE_IS_PREDEFINED(result_datatype, result_predefined); MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined); /* FIXME: refer to FIXME in MPIDI_CH3I_Shm_put_op */ diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index dac6398..c690907 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -40,10 +40,9 @@ extern void MPIDI_CH3_RMA_InitInstr(void); #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPIDI_Win_free(MPID_Win **win_ptr) { - int mpi_errno=MPI_SUCCESS, total_pt_rma_puts_accs; + int mpi_errno=MPI_SUCCESS; int in_use; MPID_Comm *comm_ptr; - int errflag = FALSE; MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FREE); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FREE); http://git.mpich.org/mpich.git/commitdiff/f31656cbb6dbfe63c985150bb4a7bd21e9... commit f31656cbb6dbfe63c985150bb4a7bd21e99b14bd Author: Xin Zhao <[email protected]> Date: Tue Jul 30 23:27:48 2013 -0500 Modify CH3 default MPIDI_Win_shared_query not to use shm_base_addrs. shm_base_addrs is allocated only when shared memory region is allocated in nemesis. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index ca9589c..dac6398 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -98,7 +98,7 @@ int MPIDI_Win_shared_query(MPID_Win *win_ptr, int target_rank, MPI_Aint *size, MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SHARED_QUERY); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SHARED_QUERY); - *(void**) baseptr = win_ptr->shm_base_addrs[0]; + *(void**) baseptr = win_ptr->base; *size = win_ptr->size; *disp_unit = win_ptr->disp_unit; http://git.mpich.org/mpich.git/commitdiff/e08bc140542eda7384e7172046312c1101... commit e08bc140542eda7384e7172046312c11010cb561 Author: Xin Zhao <[email protected]> Date: Wed Jul 31 16:19:20 2013 -0500 In MPID_Win_allocate_shared, check 'alloc_shm' info. Default set 'alloc_shm' info to TRUE. If 'alloc_shm' is set by user and value is not TRUE, throw an error. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index 731b7b9..35c1b3c 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -214,6 +214,18 @@ int MPID_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, MPID mpi_errno = win_init(size, disp_unit, MPI_WIN_FLAVOR_SHARED, MPI_WIN_UNIFIED, comm_ptr, win_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); + /* FOR ALLOCATE_SHARED, alloc_shm info is default to set to TRUE */ + (*win_ptr)->info_args.alloc_shm = TRUE; + + if (info != NULL) { + int alloc_shm_flag = 0; + char shm_alloc_value[MPI_MAX_INFO_VAL+1]; + MPIR_Info_get_impl(info, "alloc_shm", MPI_MAX_INFO_VAL, shm_alloc_value, &alloc_shm_flag); + /* if value of 'alloc_shm' info is not set to true, throw an error */ + if (alloc_shm_flag == 1 && strncmp(shm_alloc_value, "true", sizeof("true"))) + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**infoval"); + } + mpi_errno = MPIDI_CH3U_Win_fns.allocate_shared(size, disp_unit, info, comm_ptr, base_ptr, win_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); http://git.mpich.org/mpich.git/commitdiff/c58cec82601da186ee944d5151df4f5c6e... commit c58cec82601da186ee944d5151df4f5c6e734212 Author: Xin Zhao <[email protected]> Date: Thu Aug 1 14:33:22 2013 -0500 Assign new MPIDI_CH3U_Win_allocate to (*allocate_shared) so that it will be called by MPID_Win_allocate_shared. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index 080d7b5..66329cf 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -25,6 +25,7 @@ int MPIDI_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) win_fns->create = MPIDI_CH3U_Win_create; win_fns->allocate = MPIDI_CH3U_Win_allocate; + win_fns->allocate_shared = MPIDI_CH3U_Win_allocate; win_fns->create_dynamic = MPIDI_CH3U_Win_create_dynamic; MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FNS_INIT); http://git.mpich.org/mpich.git/commitdiff/a6dae684eacd22d43e275e47e015a8b000... commit a6dae684eacd22d43e275e47e015a8b00054bf7e Author: Xin Zhao <[email protected]> Date: Tue Jul 30 23:22:30 2013 -0500 Implement new MPIDI_CH3U_Win_allocate and call it through function pointer from MPID_Win_allocate. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index baf3108..080d7b5 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -214,6 +214,37 @@ int MPIDI_Win_detach(MPID_Win *win, const void *base) #undef FUNCNAME +#define FUNCNAME MPIDI_CH3U_Win_allocate +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3U_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info, + MPID_Comm *comm_ptr, void *baseptr, MPID_Win **win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE); + + if ((*win_ptr)->info_args.alloc_shm == TRUE) { + if (MPIDI_CH3U_Win_fns.allocate_shm != NULL) { + mpi_errno = MPIDI_CH3U_Win_fns.allocate_shm(size, disp_unit, info, comm_ptr, baseptr, win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + goto fn_exit; + } + } + + mpi_errno = MPIDI_CH3U_Win_allocate_no_shm(size, disp_unit, info, comm_ptr, baseptr, win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE); + return mpi_errno; + fn_fail: + goto fn_exit; +} + + +#undef FUNCNAME #define FUNCNAME MPIDI_CH3U_Win_allocate_no_shm #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index 46930d0..731b7b9 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -117,14 +117,8 @@ int MPID_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info, (*win_ptr)->info_args.alloc_shm = TRUE; } - if((*win_ptr)->info_args.alloc_shm == TRUE) { - mpi_errno = MPIDI_CH3U_Win_fns.allocate_shared(size, disp_unit, info, comm_ptr, baseptr, win_ptr); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - else { - mpi_errno = MPIDI_CH3U_Win_fns.allocate(size, disp_unit, info, comm_ptr, baseptr, win_ptr); - if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } - } + mpi_errno = MPIDI_CH3U_Win_fns.allocate(size, disp_unit, info, comm_ptr, baseptr, win_ptr); + if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } fn_fail: MPIDI_FUNC_EXIT(MPID_STATE_MPID_WIN_ALLOCATE); http://git.mpich.org/mpich.git/commitdiff/6a928e7ec1f4c27190959e0e780aec202c... commit 6a928e7ec1f4c27190959e0e780aec202cad4cd9 Author: Xin Zhao <[email protected]> Date: Thu Aug 1 15:07:15 2013 -0500 In (*allocate_shared), change type of argument 'base_ptr' from (void**) to (void*). In the code, some places use (void**)base_ptr and some places use (void*)base_ptr which are confusing. We make every win_allocate related function use (void*)base_ptr as the argument and do conversion inside the function, in order to keep the code clear. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 3e2221b..a71e8a3 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1106,7 +1106,7 @@ int MPIDI_CH3U_Comm_FinishPending( MPID_Comm * ); typedef struct { int (*create)(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *, MPID_Win **); int (*allocate)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); - int (*allocate_shared)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void **, MPID_Win **); + int (*allocate_shared)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); int (*allocate_shm)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); int (*create_dynamic)(MPID_Info *, MPID_Comm *, MPID_Win **); } MPIDI_CH3U_Win_fns_t; http://git.mpich.org/mpich.git/commitdiff/faa3a51fd280dac9df6ee538c4510dfe06... commit faa3a51fd280dac9df6ee538c4510dfe06dc0374 Author: Xin Zhao <[email protected]> Date: Tue Jul 30 23:27:12 2013 -0500 Modify MPIDI_Win_free to handle win_allocate and win_allocate_shared in CH3. Because we delete the default MPIDI_SHM_Win_free in CH3, which originally deals with win_allocate_shared in CH3, we need to modify MPIDI_Win_free to handle it. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 4382ccc..ca9589c 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -66,9 +66,10 @@ int MPIDI_Win_free(MPID_Win **win_ptr) MPIU_Free((*win_ptr)->pt_rma_puts_accs); /* Free the attached buffer for windows created with MPI_Win_allocate() */ - if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE && (*win_ptr)->size > 0) { - if ((*win_ptr)->shm_allocated != TRUE) + if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE || (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED) { + if ((*win_ptr)->shm_allocated == FALSE && (*win_ptr)->size > 0) { MPIU_Free((*win_ptr)->base); + } } MPIU_Object_release_ref(*win_ptr, &in_use); http://git.mpich.org/mpich.git/commitdiff/8bb4b63034407d8f8e85193617110b5842... commit 8bb4b63034407d8f8e85193617110b5842e25e6f Author: Xin Zhao <[email protected]> Date: Tue Jul 30 23:26:48 2013 -0500 delete CH3 default MPIDI_SHM_Win_free. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index ffc03ff..3e2221b 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1211,7 +1211,6 @@ void *MPIDI_Alloc_mem(size_t size, MPID_Info *info_ptr); int MPIDI_Free_mem(void *ptr); /* internal */ -int MPIDI_SHM_Win_free(MPID_Win **); int MPIDI_CH3I_Release_lock(MPID_Win * win_ptr); int MPIDI_CH3I_Try_acquire_win_lock(MPID_Win * win_ptr, int requested_lock); int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win *win_ptr, int source_win_hdl); diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index ddd6681..4382ccc 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -86,37 +86,6 @@ int MPIDI_Win_free(MPID_Win **win_ptr) #undef FUNCNAME -#define FUNCNAME MPIDI_SHM_Win_free -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_SHM_Win_free(MPID_Win **win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_SHM_WIN_FREE); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_SHM_WIN_FREE); - - /* Free memory allocated by the default shared memory window - implementation. Note that this implementation works only for - MPI_COMM_SELF and does not map a shared segment. */ - - MPIU_Free((*win_ptr)->base); - MPIU_Free((*win_ptr)->shm_base_addrs); - - mpi_errno = MPIDI_Win_free(win_ptr); - if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_SHM_WIN_FREE); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME #define FUNCNAME MPIDI_Win_shared_query #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) http://git.mpich.org/mpich.git/commitdiff/20ff85b8c45d03c0b3f42dda13ed718bf6... commit 20ff85b8c45d03c0b3f42dda13ed718bf64ab3fb Author: Xin Zhao <[email protected]> Date: Tue Jul 30 23:21:11 2013 -0500 delete CH3 default MPIDI_CH3U_Win_allocate_shared. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 2c1849e..ffc03ff 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1122,8 +1122,6 @@ int MPIDI_CH3U_Win_create(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *, MPID_Win **); int MPIDI_CH3U_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm, void *baseptr, MPID_Win **win); -int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info_ptr, - MPID_Comm *comm_ptr, void **baseptr, MPID_Win **win_ptr); int MPIDI_CH3U_Win_allocate_no_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, void *baseptr, MPID_Win **win_ptr); int MPIDI_CH3U_Win_create_dynamic(MPID_Info *info, MPID_Comm *comm, MPID_Win **win); diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index 9b643d2..baf3108 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -214,67 +214,6 @@ int MPIDI_Win_detach(MPID_Win *win, const void *base) #undef FUNCNAME -#define FUNCNAME MPIDI_CH3U_Win_allocate_shared -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, - void **base_ptr, MPID_Win **win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Comm *comm_self_ptr = NULL; - MPID_Group *group_comm, *group_self; - int result; - MPIU_CHKPMEM_DECL(1); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE_SHARED); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE_SHARED); - -#ifdef HAVE_ERROR_CHECKING - /* The baseline CH3 implementation only works with MPI_COMM_SELF */ - MPID_Comm_get_ptr( MPI_COMM_SELF, comm_self_ptr ); - - mpi_errno = MPIR_Comm_group_impl(comm_ptr, &group_comm); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - mpi_errno = MPIR_Comm_group_impl(comm_self_ptr, &group_self); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - mpi_errno = MPIR_Group_compare_impl(group_comm, group_self, &result); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - mpi_errno = MPIR_Group_free_impl(group_comm); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - mpi_errno = MPIR_Group_free_impl(group_self); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - - if (result != MPI_IDENT) { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_RMA_SHARED, "**ch3|win_shared_comm"); - } -#endif - - mpi_errno = MPIDI_CH3U_Win_allocate(size, disp_unit, info, comm_ptr, - base_ptr, win_ptr); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - - MPIU_CHKPMEM_MALLOC((*win_ptr)->shm_base_addrs, void **, - 1 /* comm_size */ * sizeof(void *), - mpi_errno, "(*win_ptr)->shm_base_addrs"); - - (*win_ptr)->shm_base_addrs[0] = *base_ptr; - - /* Register the shared memory window free function, which will free the - memory allocated here. */ - (*win_ptr)->RMAFns.Win_free = MPIDI_SHM_Win_free; - -fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE_SHARED); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ -fn_fail: - MPIU_CHKPMEM_REAP(); - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME #define FUNCNAME MPIDI_CH3U_Win_allocate_no_shm #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) http://git.mpich.org/mpich.git/commitdiff/3e23947322ee6257e66eb7d316de01ffbd... commit 3e23947322ee6257e66eb7d316de01ffbda3ec3c Author: Xin Zhao <[email protected]> Date: Tue Jul 30 23:17:45 2013 -0500 In CH3 layer, change function name from MPIDI_CH3U_Win_allocate to MPIDI_CH3U_Win_allocate_no_shm. Also add prototype in header file. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index eb21fff..2c1849e 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1124,6 +1124,8 @@ int MPIDI_CH3U_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm, void *baseptr, MPID_Win **win); int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info_ptr, MPID_Comm *comm_ptr, void **baseptr, MPID_Win **win_ptr); +int MPIDI_CH3U_Win_allocate_no_shm(MPI_Aint size, int disp_unit, MPID_Info *info, + MPID_Comm *comm_ptr, void *baseptr, MPID_Win **win_ptr); int MPIDI_CH3U_Win_create_dynamic(MPID_Info *info, MPID_Comm *comm, MPID_Win **win); diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index 988ae95..9b643d2 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -275,18 +275,18 @@ fn_fail: #undef FUNCNAME -#define FUNCNAME MPIDI_CH3U_Win_allocate +#define FUNCNAME MPIDI_CH3U_Win_allocate_no_shm #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3U_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info, - MPID_Comm *comm_ptr, void *baseptr, MPID_Win **win_ptr ) +int MPIDI_CH3U_Win_allocate_no_shm(MPI_Aint size, int disp_unit, MPID_Info *info, + MPID_Comm *comm_ptr, void *baseptr, MPID_Win **win_ptr ) { void **base_pp = (void **) baseptr; int mpi_errno = MPI_SUCCESS; MPIU_CHKPMEM_DECL(1); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE_NO_SHM); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE_NO_SHM); if (size > 0) { MPIU_CHKPMEM_MALLOC(*base_pp, void *, size, mpi_errno, "(*win_ptr)->base"); @@ -302,7 +302,7 @@ int MPIDI_CH3U_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info, if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_WIN_ALLOCATE_NO_SHM); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: http://git.mpich.org/mpich.git/commitdiff/0a338e5dbe6a3589a9da8ba467d4bc18d3... commit 0a338e5dbe6a3589a9da8ba467d4bc18d3f94270 Author: Xin Zhao <[email protected]> Date: Thu Aug 1 14:44:22 2013 -0500 In nemesis layer, change type of argument 'base_ptr' from (void**) to (void*). In the code, some places use (void**)base_ptr and some places use (void*)base_ptr which are confusing. We make every win_allocate related function use (void*)base_ptr as the argument and do conversion inside the function, in order to keep the code clear. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index eecf8fe..9d9c5ab 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -19,7 +19,7 @@ MPIU_INSTR_DURATION_EXTERN_DECL(wincreate_allgather); #endif static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, - void **base_ptr, MPID_Win **win_ptr); + void *base_ptr, MPID_Win **win_ptr); #undef FUNCNAME #define FUNCNAME MPIDI_CH3_Win_fns_init @@ -45,9 +45,10 @@ int MPIDI_CH3_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, - MPID_Comm *comm_ptr, void **base_ptr, MPID_Win **win_ptr) + MPID_Comm *comm_ptr, void *base_ptr, MPID_Win **win_ptr) { int mpi_errno = MPI_SUCCESS; + void **base_pp = (void **) base_ptr; int i, k, comm_size, rank; int node_size, node_rank; MPID_Comm *node_comm_ptr; @@ -342,7 +343,7 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * for (i = 0; i < comm_size; ++i) (*win_ptr)->base_addrs[i] = MPIU_AintToPtr(tmp_buf[i]); - *base_ptr = (*win_ptr)->base; + *base_pp = (*win_ptr)->base; /* Provide operation overrides for this window flavor */ (*win_ptr)->RMAFns.Win_shared_query = MPIDI_CH3_SHM_Win_shared_query; http://git.mpich.org/mpich.git/commitdiff/48aaa82295c800f976c49a7aaba4344175... commit 48aaa82295c800f976c49a7aaba4344175d643b5 Author: Xin Zhao <[email protected]> Date: Tue Jul 30 22:25:25 2013 -0500 In nemesis layer, change function name from MPIDI_CH3I_Win_allocate_shared to MPIDI_CH3I_Win_allocate_shm. MPIDI_CH3I_Win_allocate_shm is assigned to (*allocate_shm). Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index 57bbb3f..eecf8fe 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -18,8 +18,8 @@ MPIU_INSTR_DURATION_EXTERN_DECL(wincreate_allgather); #endif -static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, - void **base_ptr, MPID_Win **win_ptr); +static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, + void **base_ptr, MPID_Win **win_ptr); #undef FUNCNAME #define FUNCNAME MPIDI_CH3_Win_fns_init @@ -41,11 +41,11 @@ int MPIDI_CH3_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Win_allocate_shared +#define FUNCNAME MPIDI_CH3I_Win_allocate_shm #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, - MPID_Comm *comm_ptr, void **base_ptr, MPID_Win **win_ptr) +static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, + MPID_Comm *comm_ptr, void **base_ptr, MPID_Win **win_ptr) { int mpi_errno = MPI_SUCCESS; int i, k, comm_size, rank; @@ -58,9 +58,9 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Inf int noncontig = FALSE; MPIU_CHKPMEM_DECL(6); MPIU_CHKLMEM_DECL(3); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHARED); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHM); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHARED); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHM); /* If create flavor is MPI_WIN_FLAVOR_ALLOCATE, alloc_shared_noncontig is set to 1 by default. */ if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE) @@ -350,7 +350,7 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Inf fn_exit: MPIU_CHKLMEM_FREEALL(); - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHARED); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHM); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: http://git.mpich.org/mpich.git/commitdiff/723e5d5e86d33e5c668dfc3b528618a885... commit 723e5d5e86d33e5c668dfc3b528618a88587b928 Author: Xin Zhao <[email protected]> Date: Wed Jul 31 16:04:33 2013 -0500 add function pointer 'allocate_shm' to win_fns. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index 3a4c591..57bbb3f 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -32,6 +32,8 @@ int MPIDI_CH3_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3_WIN_FNS_INIT); + win_fns->allocate_shm = MPIDI_CH3I_Win_allocate_shm; + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3_WIN_FNS_INIT); return mpi_errno; diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 1350e64..eb21fff 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1107,6 +1107,7 @@ typedef struct { int (*create)(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *, MPID_Win **); int (*allocate)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); int (*allocate_shared)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void **, MPID_Win **); + int (*allocate_shm)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); int (*create_dynamic)(MPID_Info *, MPID_Comm *, MPID_Win **); } MPIDI_CH3U_Win_fns_t; http://git.mpich.org/mpich.git/commitdiff/8bf8f815a5105af42fb368f5da4b718546... commit 8bf8f815a5105af42fb368f5da4b71854620d90f Author: Xin Zhao <[email protected]> Date: Wed Jul 31 16:02:55 2013 -0500 delete assignment of (*allocate_shared) in CH3 and nemesis. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index c867138..3a4c591 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -32,8 +32,6 @@ int MPIDI_CH3_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3_WIN_FNS_INIT); - win_fns->allocate_shared = MPIDI_CH3I_Win_allocate_shared; - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3_WIN_FNS_INIT); return mpi_errno; diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index ef7d84c..988ae95 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -25,7 +25,6 @@ int MPIDI_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) win_fns->create = MPIDI_CH3U_Win_create; win_fns->allocate = MPIDI_CH3U_Win_allocate; - win_fns->allocate_shared = MPIDI_CH3U_Win_allocate_shared; win_fns->create_dynamic = MPIDI_CH3U_Win_create_dynamic; MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FNS_INIT); http://git.mpich.org/mpich.git/commitdiff/c7bc4694fbb0b10f6a09141794c6e0664a... commit c7bc4694fbb0b10f6a09141794c6e0664a76197a Author: Xin Zhao <[email protected]> Date: Wed Jul 31 15:48:16 2013 -0500 When judging if origin and target process are on the same node, using vc->node_id flag instead of vc->ch.is_local flag. Flag 'is_local' is not correct because it is defined in nemesis, not in CH3. Flag 'node_id' is defined in CH3. Note that for ch3:sock, even if origin and target are on the same node, they are not within the same SHM region. Currently ch3:sock is filtered out by checking shm_allocated flag first. In future we need to figure out a way to check if origin and target are within the same "SHM comm". Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index b0d0df1..e469edc 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -325,16 +325,22 @@ static inline int MPIDI_CH3I_Shm_put_op(const void *origin_addr, int origin_coun MPID_Win *win_ptr) { int mpi_errno = MPI_SUCCESS; - MPIDI_VC_t *vc = NULL; void *base = NULL; int disp_unit; + MPIDI_VC_t *orig_vc, *target_vc; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP); - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on + the same node. However, in ch3:sock, even if origin and target are on the same node, they do + not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, + which is only set to TRUE when SHM region is allocated in nemesis. + In future we need to figure out a way to check if origin and target are in the same "SHM comm". + */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; } @@ -372,7 +378,7 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun int origin_predefined, target_predefined; MPI_User_function *uop = NULL; MPID_Datatype *dtp; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; int mpi_errno = MPI_SUCCESS; MPIU_CHKLMEM_DECL(2); MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP); @@ -382,9 +388,10 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, origin_predefined); MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined); - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + /* FIXME: refer to FIXME in MPIDI_CH3I_Shm_put_op */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { shm_op = 1; base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; @@ -527,7 +534,7 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ MPI_User_function *uop = NULL; MPID_Datatype *dtp; int origin_predefined, result_predefined, target_predefined; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; int mpi_errno = MPI_SUCCESS; MPIU_CHKLMEM_DECL(2); MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP); @@ -541,9 +548,10 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ MPIDI_CH3I_DATATYPE_IS_PREDEFINED(result_datatype, result_predefined); MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined); - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + /* FIXME: refer to FIXME in MPIDI_CH3I_Shm_put_op */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); @@ -699,15 +707,16 @@ static inline int MPIDI_CH3I_Shm_get_op(void *origin_addr, int origin_count, MPI { void *base = NULL; int disp_unit; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_OP); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_GET_OP); - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + /* FIXME: refer to FIXME in MPIDI_CH3I_Shm_put_op */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; } @@ -742,15 +751,16 @@ static inline int MPIDI_CH3I_Shm_cas_op(const void *origin_addr, const void *com void *base = NULL, *dest_addr = NULL; int disp_unit; int len, shm_locked = 0; - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; + MPIDI_VC_t *orig_vc, *target_vc; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP); - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + /* FIXME: refer to FIXME in MPIDI_CH3I_Shm_put_op */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; @@ -801,15 +811,16 @@ static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_ad MPI_User_function *uop = NULL; int disp_unit; int len, one, shm_locked = 0; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP); - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + /* FIXME: refer to FIXME in MPIDI_CH3I_Shm_put_op */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; diff --git a/src/mpid/ch3/src/ch3u_rma_acc_ops.c b/src/mpid/ch3/src/ch3u_rma_acc_ops.c index 0253095..e8e3dea 100644 --- a/src/mpid/ch3/src/ch3u_rma_acc_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_acc_ops.c @@ -71,7 +71,7 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; /* Append the operation to the window's RMA ops queue */ MPIU_INSTR_DURATION_START(rmaqueue_alloc); @@ -103,8 +103,10 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, because this operation will be directly done on shared memory region, instead of sending and receiving through the progress engine, therefore datatype will not be referenced by the progress engine */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if (!(win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (!(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { /* if source or target datatypes are derived, increment their reference counts */ if (!origin_predefined) { diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index b7d6d49..ddd6681 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -190,7 +190,7 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; /* queue it up */ MPIU_INSTR_DURATION_START(rmaqueue_alloc); @@ -218,8 +218,16 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype because this operation will be directly done on shared memory region, instead of sending and receiving through the progress engine, therefore datatype will not be referenced by the progress engine */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if (!(win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + + /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on + the same node. However, in ch3:sock, even if origin and target are on the same node, they do + not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, + which is only set to TRUE when SHM region is allocated in nemesis. + In future we need to figure out a way to check if origin and target are in the same "SHM comm". + */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (!(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { /* if source or target datatypes are derived, increment their reference counts */ MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, predefined); @@ -297,7 +305,7 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; /* queue it up */ MPIU_INSTR_DURATION_START(rmaqueue_alloc); @@ -322,8 +330,10 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype because this operation will be directly done on shared memory region, instead of sending and receiving through the progress engine, therefore datatype will not be referenced by the progress engine */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if (!(win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (!(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { /* if source or target datatypes are derived, increment their reference counts */ MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, predefined); @@ -407,7 +417,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; /* queue it up */ MPIU_INSTR_DURATION_START(rmaqueue_alloc); @@ -453,8 +463,10 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype because this operation will be directly done on shared memory region, instead of sending and receiving through the progress engine, therefore datatype will not be referenced by the progress engine */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if (!(win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + if (!(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { /* if source or target datatypes are derived, increment their reference counts */ if (!origin_predefined) diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index df097d0..c16484a 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -329,6 +329,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr) { int nRequest = 0; int nRequestNew = 0; + MPIDI_VC_t *orig_vc, *target_vc; MPIU_INSTR_DURATION_START(winfence_rs); /* This is the second or later fence. Do all the preceding RMA ops. */ comm_ptr = win_ptr->comm_ptr; @@ -356,11 +357,11 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr) ops from this process */ total_op_count = 0; curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); while (curr_ptr != NULL) { - MPIDI_VC_t *vc = NULL; - MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &vc); - if (!(win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &target_vc); + if (!(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { total_op_count++; rma_target_proc[curr_ptr->target_rank] = 1; nops_to_proc[curr_ptr->target_rank]++; @@ -400,9 +401,8 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr) curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); while (curr_ptr != NULL) { - MPIDI_VC_t *vc = NULL; - MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &vc); - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { MPIDI_CH3I_DO_SHM_OP(curr_ptr, win_ptr, mpi_errno); MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr); } @@ -1576,6 +1576,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr) int start_grp_size, *ranks_in_start_grp, *ranks_in_win_grp, rank; int nRequest = 0; int nRequestNew = 0; + MPIDI_VC_t *orig_vc, *target_vc; MPIU_CHKLMEM_DECL(9); MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_COMPLETE); @@ -1688,11 +1689,11 @@ int MPIDI_Win_complete(MPID_Win *win_ptr) total_op_count = 0; curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); while (curr_ptr != NULL) { - MPIDI_VC_t *vc = NULL; - MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &vc); - if (!(win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &target_vc); + if (!(win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { nops_to_proc[curr_ptr->target_rank]++; total_op_count++; } @@ -1714,9 +1715,8 @@ int MPIDI_Win_complete(MPID_Win *win_ptr) curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); while (curr_ptr != NULL) { - MPIDI_VC_t *vc = NULL; - MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &vc); - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + MPIDI_Comm_get_vc(win_ptr->comm_ptr, curr_ptr->target_rank, &target_vc); + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { MPIDI_CH3I_DO_SHM_OP(curr_ptr, win_ptr, mpi_errno); MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr); } @@ -2584,7 +2584,7 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr, int target_rank, MPIDI_RMA_Op_t *curr_ptr; MPI_Win source_win_handle = MPI_WIN_NULL, target_win_handle = MPI_WIN_NULL; int nRequest=0, nRequestNew=0; - MPIDI_VC_t *vc = NULL; + MPIDI_VC_t *orig_vc, *target_vc; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_DO_PASSIVE_TARGET_RMA); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_DO_PASSIVE_TARGET_RMA); @@ -2594,12 +2594,13 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr, int target_rank, (win_ptr->targets[target_rank].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED && win_ptr->targets[target_rank].remote_lock_assert & MPI_MODE_NOCHECK)); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + /* if alloc_shm is enabled and target process is on the same node, directly perform RMA operations at the origin side and remove them from passive RMA operation list */ - - MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { + if (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) { curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list); while (curr_ptr != NULL) { MPIU_Assert(curr_ptr->target_rank == target_rank); http://git.mpich.org/mpich.git/commitdiff/a64a1d2ce36c1b0bbf4779734a67aa6643... commit a64a1d2ce36c1b0bbf4779734a67aa6643192f30 Author: Xin Zhao <[email protected]> Date: Wed Jul 31 15:44:48 2013 -0500 In SHM RMA operations, delete using MPI_WIN_FLAVOR_SHARED to judge if using shm_base_addrs. Because if MPI_WIN_FLAVOR_SHARED is used in ch3:sock, it will allocate normal memory instead of shared memory, therefore shm_base_addrs will not be used. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 3221c6a..b0d0df1 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -334,8 +334,7 @@ static inline int MPIDI_CH3I_Shm_put_op(const void *origin_addr, int origin_coun MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if ((win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) || - (win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; } @@ -385,8 +384,7 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if ((win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) || - (win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { shm_op = 1; base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; @@ -545,8 +543,7 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if ((win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) || - (win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); @@ -710,8 +707,7 @@ static inline int MPIDI_CH3I_Shm_get_op(void *origin_addr, int origin_count, MPI MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if ((win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) || - (win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; } @@ -754,8 +750,7 @@ static inline int MPIDI_CH3I_Shm_cas_op(const void *origin_addr, const void *com MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if ((win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) || - (win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; @@ -814,8 +809,7 @@ static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_ad MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &vc); - if ((win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) || - (win_ptr->shm_allocated == TRUE && vc->ch.is_local)) { + if (win_ptr->shm_allocated == TRUE && vc->ch.is_local) { base = win_ptr->shm_base_addrs[target_rank]; disp_unit = win_ptr->disp_units[target_rank]; http://git.mpich.org/mpich.git/commitdiff/66221968523896310b3920cbfd44f4c1fd... commit 66221968523896310b3920cbfd44f4c1fd9bc205 Author: Xin Zhao <[email protected]> Date: Tue Jul 30 21:12:31 2013 -0500 Move shm_allocated flag from nemesis layer to CH3 layer. Because CH3 layer needs to know if shared memory region is allocated in lower layer. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/channels/nemesis/include/mpid_nem_pre.h b/src/mpid/ch3/channels/nemesis/include/mpid_nem_pre.h index 42cb894..b954b44 100644 --- a/src/mpid/ch3/channels/nemesis/include/mpid_nem_pre.h +++ b/src/mpid/ch3/channels/nemesis/include/mpid_nem_pre.h @@ -18,8 +18,6 @@ typedef pthread_mutex_t MPIDI_CH3I_SHM_MUTEX; #define MPIDI_CH3_WIN_DECL \ - int shm_allocated; /* flag: TRUE iff this window has a shared memory \ - region associated with it */ \ void *shm_base_addr; /* base address of shared memory region */ \ MPI_Aint shm_segment_len; /* size of shared memory region */ \ MPIU_SHMW_Hnd_t shm_segment_handle; /* handle to shared memory region */ \ diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 8210c97..3e0935d 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -303,6 +303,8 @@ struct MPIDI_Win_target_state { all processes remains consistent. */ \ MPID_Group *start_group_ptr; /* group passed in MPI_Win_start */ \ int start_assert; /* assert passed to MPI_Win_start */ \ + int shm_allocated; /* flag: TRUE iff this window has a shared memory \ + region associated with it */ \ #ifdef MPIDI_CH3_WIN_DECL #define MPID_DEV_WIN_DECL \ diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index d330933..46930d0 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -294,6 +294,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->epoch_state = MPIDI_EPOCH_NONE; (*win_ptr)->epoch_count = 0; (*win_ptr)->at_rma_ops_list = NULL; + (*win_ptr)->shm_allocated = FALSE; /* Initialize the passive target lock state */ MPIU_CHKPMEM_MALLOC((*win_ptr)->targets, struct MPIDI_Win_target_state *, ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/include/mpid_nem_pre.h | 2 - src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c | 21 +++--- src/mpid/ch3/include/mpidimpl.h | 8 +- src/mpid/ch3/include/mpidpre.h | 2 + src/mpid/ch3/include/mpidrma.h | 68 +++++++++-------- src/mpid/ch3/src/ch3u_rma_acc_ops.c | 8 +- src/mpid/ch3/src/ch3u_rma_ops.c | 71 +++++++----------- src/mpid/ch3/src/ch3u_rma_sync.c | 33 ++++---- src/mpid/ch3/src/ch3u_win_fns.c | 78 ++++++-------------- src/mpid/ch3/src/mpid_rma.c | 23 ++++-- 10 files changed, 140 insertions(+), 174 deletions(-) hooks/post-receive -- MPICH primary repository