November 2014 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.1.3-78-gfc7617f
by noreply＠mpich.org 03 Nov '14

03 Nov '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via fc7617f21a4247b75f7989ecd8063d2b3b29ae7e (commit) via b1685139330580132c64e4b616ae91824b9bf5b8 (commit) via ba1a400c4de2df8a662f0f9f856d824ed05ec221 (commit) via 0eaf344b95ae93d0546c4a20f4c6bc0581a88e51 (commit) via cb04acb30ecb4e244973fa7c8c6033f92c71c89d (commit) via 61f952c7a7f9ec2c59e496bf8275b29f12132768 (commit) via 5c51303260595939cb15d658e5f369bc148c9a4a (commit) from 0086b7bf7f2ec4e4ef39bca1c153c1000e137a3c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/fc7617f21a4247b75f7989ecd8063d2b3… commit fc7617f21a4247b75f7989ecd8063d2b3b29ae7e Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Nov 2 22:00:27 2014 -0600 Add global / local pools of RMA ops and related APIs. Instead of allocating / deallocating RMA operations whenever an RMA op is posted by user, we allocate fixed size operation pools beforehand and take the op element from those pools when an RMA op is posted. With only a local (per-window) op pool, the number of ops allocated can increase arbitrarily if many windows are created. Alternatively, if we only use a global op pool, other windows might use up all operations thus starving the window we are working on. In this patch we create two pools: a local (per-window) pool and a global pool. Every window is guaranteed to have at least the number of operations in the local pool. If we run out of these operations, we check in the global pool to see if we have any operations left. When an operation is released, it is added back to the same pool it was allocated from. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index bc9fa21..bc64150 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -10,6 +10,66 @@ #include "mpl_utlist.h" #include "mpid_rma_types.h" +extern struct MPIDI_RMA_Op *global_rma_op_pool, *global_rma_op_pool_tail, *global_rma_op_pool_start; + +/* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and + * initialize it. If we cannot get one, return NULL. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Win_op_alloc +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr) +{ + MPIDI_RMA_Op_t *e; + + if (win_ptr->op_pool == NULL) { + /* local pool is empty, try to find something in the global pool */ + if (global_rma_op_pool == NULL) + return NULL; + else { + e = global_rma_op_pool; + MPL_LL_DELETE(global_rma_op_pool, global_rma_op_pool_tail, e); + } + } + else { + e = win_ptr->op_pool; + MPL_LL_DELETE(win_ptr->op_pool, win_ptr->op_pool_tail, e); + } + + e->dataloop = NULL; + e->request = NULL; + + return e; +} + +/* MPIDI_CH3I_Win_op_free(): put an op element back to the op pool which + * it belongs to. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Win_op_free +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_Win_op_free(MPID_Win * win_ptr, MPIDI_RMA_Op_t * e) +{ + int mpi_errno = MPI_SUCCESS; + + /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */ + if (e->dataloop != NULL) { + MPIU_Free(e->dataloop); + } + + /* We enqueue elements to the right pool, so when they get freed + * at window free time, they won't conflict with the global pool + * or other windows */ + /* use PREPEND when return objects back to the pool + in order to improve cache performance */ + if (e->pool_type == MPIDI_RMA_POOL_WIN) + MPL_LL_PREPEND(win_ptr->op_pool, win_ptr->op_pool_tail, e); + else + MPL_LL_PREPEND(global_rma_op_pool, global_rma_op_pool_tail, e); + + return mpi_errno; +} + /* Return nonzero if the RMA operations list is empty. */ #undef FUNCNAME @@ -40,9 +100,9 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * lis #define FUNCNAME MPIDI_CH3I_RMA_Ops_tail #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list) +static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list_tail) { - return (*list) ? (*list)->prev : NULL; + return (*list_tail); } @@ -55,9 +115,10 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * lis #define FUNCNAME MPIDI_CH3I_RMA_Ops_append #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) +static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Ops_list_t * list_tail, + MPIDI_RMA_Op_t * elem) { - MPL_DL_APPEND(*list, elem); + MPL_LL_APPEND(*list, *list_tail, elem); } @@ -71,30 +132,23 @@ static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_ #define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list, +static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Ops_list_t * list_tail, MPIDI_RMA_Op_t ** new_elem) { int mpi_errno = MPI_SUCCESS; MPIDI_RMA_Op_t *tmp_ptr; - MPIU_CHKPMEM_DECL(1); - /* FIXME: We should use a pool allocator here */ - MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t), - mpi_errno, "RMA operation entry"); + tmp_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr); + MPIU_ERR_CHKANDJUMP(tmp_ptr == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem"); - tmp_ptr->next = NULL; - tmp_ptr->dataloop = NULL; - tmp_ptr->request = NULL; - - MPL_DL_APPEND(*list, tmp_ptr); + MPL_LL_APPEND(*list, *list_tail, tmp_ptr); *new_elem = tmp_ptr; fn_exit: - MPIU_CHKPMEM_COMMIT(); return mpi_errno; fn_fail: - MPIU_CHKPMEM_REAP(); *new_elem = NULL; goto fn_exit; } @@ -109,9 +163,10 @@ static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list, #define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) +static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Ops_list_t *list_tail, + MPIDI_RMA_Op_t * elem) { - MPL_DL_DELETE(*list, elem); + MPL_LL_DELETE(*list, *list_tail, elem); } @@ -124,19 +179,17 @@ static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_ #define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list, +static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Ops_list_t * list_tail, MPIDI_RMA_Op_t * curr_ptr) { MPIDI_RMA_Op_t *tmp_ptr = curr_ptr; MPIU_Assert(curr_ptr != NULL); - MPL_DL_DELETE(*list, curr_ptr); + MPL_LL_DELETE(*list, *list_tail, curr_ptr); - /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */ - if (tmp_ptr->dataloop != NULL) - MPIU_Free(tmp_ptr->dataloop); - MPIU_Free(tmp_ptr); + MPIDI_CH3I_Win_op_free(win_ptr, tmp_ptr); } @@ -151,12 +204,13 @@ static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list, #define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list, +static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Ops_list_t * list_tail, MPIDI_RMA_Op_t ** curr_ptr) { MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next; - MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr); + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, list, list_tail, *curr_ptr); *curr_ptr = next_ptr; } @@ -167,12 +221,13 @@ static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list, #define FUNCNAME MPIDI_CH3I_RMA_Ops_free #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list) +static inline void MPIDI_CH3I_RMA_Ops_free(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Ops_list_t * list_tail) { MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr; - MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) { - MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr); + MPL_LL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) { + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, list, list_tail, curr_ptr); } } @@ -197,4 +252,19 @@ static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_p } } +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list_tail(MPID_Win * win_ptr, int target) +{ + if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE || + win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) { + return &win_ptr->at_rma_ops_list_tail; + } + else { + return &win_ptr->targets[target].rma_ops_list_tail; + } +} + #endif /* MPID_RMA_OPLIST_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index c95847c..0176c96 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -46,9 +46,13 @@ typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */ int has_sticky_ub, has_sticky_lb; } MPIDI_RMA_dtype_info; +typedef enum MPIDI_RMA_Pool_type { + MPIDI_RMA_POOL_WIN = 6, + MPIDI_RMA_POOL_GLOBAL = 7 +} MPIDI_RMA_Pool_type_t; + /* for keeping track of RMA ops, which will be executed at the next sync call */ typedef struct MPIDI_RMA_Op { - struct MPIDI_RMA_Op *prev; /* pointer to next element in list */ struct MPIDI_RMA_Op *next; /* pointer to next element in list */ void *origin_addr; @@ -69,6 +73,7 @@ typedef struct MPIDI_RMA_Op { int target_rank; MPIDI_CH3_Pkt_t pkt; + MPIDI_RMA_Pool_type_t pool_type; } MPIDI_RMA_Op_t; typedef struct MPIDI_PT_single_op { diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 3d59636..a16f003 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -249,6 +249,7 @@ struct MPIDI_RMA_op; /* forward decl from mpidrma.h */ struct MPIDI_Win_target_state { struct MPIDI_RMA_Op *rma_ops_list; /* List of outstanding RMA operations */ + struct MPIDI_RMA_Op *rma_ops_list_tail; volatile enum MPIDI_CH3_Lock_states remote_lock_state; /* Indicates the state of the target process' "lock" for passive target @@ -284,6 +285,7 @@ struct MPIDI_Win_target_state { mode of operation */ \ struct MPIDI_RMA_Op *at_rma_ops_list; /* Ops list for active target \ mode of operation. */ \ + struct MPIDI_RMA_Op *at_rma_ops_list_tail; \ enum MPIDI_Win_epoch_states epoch_state; \ int epoch_count; \ int fence_issued; /* Indicates if fence has been called, and if an \ @@ -296,6 +298,9 @@ struct MPIDI_Win_target_state { int start_assert; /* assert passed to MPI_Win_start */ \ int shm_allocated; /* flag: TRUE iff this window has a shared memory \ region associated with it */ \ + struct MPIDI_RMA_Op *op_pool_start; /* start pointer used for freeing */\ + struct MPIDI_RMA_Op *op_pool; /* pool of operations */ \ + struct MPIDI_RMA_Op *op_pool_tail; /* tail pointer to pool of operations. */ \ #ifdef MPIDI_CH3_WIN_DECL #define MPID_DEV_WIN_DECL \ @@ -440,4 +445,8 @@ MPID_REQUEST_DECL /* Tell the RMA code to use a table of RMA functions provided by the ADI */ #define USE_MPID_RMA_TABLE + +int MPIDI_RMA_init(void); +void MPIDI_RMA_finalize(void); + #endif /* !defined(MPICH_MPIDPRE_H_INCLUDED) */ diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index f45a2b4..a9ee171 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -73,11 +73,12 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_put_t *put_pkt = NULL; /* queue it up */ - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -184,11 +185,12 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; /* queue it up */ - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -296,11 +298,12 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; /* queue it up */ - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -441,10 +444,11 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; /* Append the operation to the window's RMA ops queue */ - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -575,11 +579,12 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL; /* Append this operation to the RMA ops queue */ - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -669,11 +674,12 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Ops_list_t *ops_list_tail = MPIDI_CH3I_RMA_Get_ops_list_tail(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_fop_t *fop_pkt = NULL; /* Append this operation to the RMA ops queue */ - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 3629dd5..b63e956 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -349,9 +349,11 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, int *wait_for_rma_done_pkt, MPIDI_CH3_Pkt_flags_t sync_flags); static int send_lock_put_or_acc(MPID_Win *, int); static int send_lock_get(MPID_Win *, int); -static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list); +static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list, + MPIDI_RMA_Ops_list_t * ops_list_tail); static inline int rma_list_gc(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list, + MPIDI_RMA_Ops_list_t * ops_list_tail, MPIDI_RMA_Op_t * last_elm, int *nDone); @@ -366,6 +368,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) int *rma_target_proc, *nops_to_proc, i, total_op_count, *curr_ops_cnt; MPIDI_RMA_Op_t *curr_ptr; MPIDI_RMA_Ops_list_t *ops_list; + MPIDI_RMA_Ops_list_t *ops_list_tail; MPID_Comm *comm_ptr; MPID_Progress_state progress_state; int errflag = FALSE; @@ -432,6 +435,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) /* Note, active target uses the following ops list, and passive * target uses win_ptr->targets[..] */ ops_list = &win_ptr->at_rma_ops_list; + ops_list_tail = &win_ptr->at_rma_ops_list_tail; /* set rma_target_proc[i] to 1 if rank i is a target of RMA * ops from this process */ @@ -494,7 +498,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) curr_ops_cnt[curr_ptr->target_rank]++; /* If the request is null, we can remove it immediately */ if (!curr_ptr->request) { - MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr); + MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, ops_list, ops_list_tail, &curr_ptr); } else { nRequest++; @@ -511,7 +515,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone); + mpi_errno = rma_list_gc(win_ptr, ops_list, ops_list_tail, curr_ptr, &nDone); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); /* if (nDone > 0) printf("nDone = %d\n", nDone); */ @@ -533,7 +537,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) * needed while looping through the requests. */ if (total_op_count) { - mpi_errno = rma_list_complete(win_ptr, ops_list); + mpi_errno = rma_list_complete(win_ptr, ops_list, ops_list_tail); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -927,6 +931,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) int i, j, dst, total_op_count, *curr_ops_cnt; MPIDI_RMA_Op_t *curr_ptr; MPIDI_RMA_Ops_list_t *ops_list; + MPIDI_RMA_Ops_list_t *ops_list_tail; MPID_Comm *comm_ptr; int start_grp_size, *ranks_in_win_grp, rank; int nRequest = 0; @@ -986,6 +991,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) /* Note, active target uses the following ops list, and passive * target uses win_ptr->targets[..] */ ops_list = &win_ptr->at_rma_ops_list; + ops_list_tail = &win_ptr->at_rma_ops_list_tail; MPIU_CHKLMEM_MALLOC(nops_to_proc, int *, comm_size * sizeof(int), mpi_errno, "nops_to_proc"); for (i = 0; i < comm_size; i++) @@ -1026,7 +1032,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) curr_ops_cnt[curr_ptr->target_rank]++; /* If the request is null, we can remove it immediately */ if (!curr_ptr->request) { - MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr); + MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, ops_list, ops_list_tail, &curr_ptr); } else { nRequest++; @@ -1037,7 +1043,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone); + mpi_errno = rma_list_gc(win_ptr, ops_list, ops_list_tail, curr_ptr, &nDone); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); nRequest -= nDone; @@ -1086,7 +1092,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) if (request) { MPIDI_RMA_Op_t *new_ptr = NULL; - mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(win_ptr, ops_list, ops_list_tail, &new_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1099,7 +1105,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) } if (new_total_op_count) { - mpi_errno = rma_list_complete(win_ptr, ops_list); + mpi_errno = rma_list_complete(win_ptr, ops_list, ops_list_tail); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -1921,7 +1927,7 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, *wait_for_rma_done_pkt = 0; } else { - MPIDI_RMA_Op_t *tail = MPIDI_CH3I_RMA_Ops_tail(&win_ptr->targets[target_rank].rma_ops_list); + MPIDI_RMA_Op_t *tail = MPIDI_CH3I_RMA_Ops_tail(&win_ptr->targets[target_rank].rma_ops_list_tail); /* Check if we can piggyback the RMA done acknowlegdement on the last * operation in the epoch. */ @@ -1948,8 +1954,10 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, *wait_for_rma_done_pkt = 0; MPIDI_CH3I_RMA_Ops_unlink(&win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail, curr_ptr); MPIDI_CH3I_RMA_Ops_append(&win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail, curr_ptr); break; } @@ -2023,7 +2031,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, /* If the request is null, we can remove it immediately */ if (!curr_ptr->request) { - MPIDI_CH3I_RMA_Ops_free_and_next(&win_ptr->targets[target_rank].rma_ops_list, + MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail, &curr_ptr); } else { @@ -2036,7 +2045,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); mpi_errno = - rma_list_gc(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, curr_ptr, + rma_list_gc(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail, curr_ptr, &nDone); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -2048,7 +2058,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, } if (nops) { - mpi_errno = rma_list_complete(win_ptr, &win_ptr->targets[target_rank].rma_ops_list); + mpi_errno = rma_list_complete(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -2337,7 +2348,8 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) /* Free MPIDI_RMA_Ops_list - the lock packet should still be in place, so * we have to free two elements. */ - MPIDI_CH3I_RMA_Ops_free(&win_ptr->targets[target_rank].rma_ops_list); + MPIDI_CH3I_RMA_Ops_free(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail); fn_fail: MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_PUT_OR_ACC); @@ -2450,7 +2462,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) /* Free MPIDI_RMA_Ops_list - the lock packet should still be in place, so * we have to free two elements. */ - MPIDI_CH3I_RMA_Ops_free(&win_ptr->targets[target_rank].rma_ops_list); + MPIDI_CH3I_RMA_Ops_free(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, + &win_ptr->targets[target_rank].rma_ops_list_tail); fn_fail: MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_GET); @@ -2460,7 +2473,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) /* ------------------------------------------------------------------------ */ /* list_complete_timer/counter and list_block_timer defined above */ -static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list) +static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list, + MPIDI_RMA_Ops_list_t *ops_list_tail) { int ntimes = 0, mpi_errno = 0; MPIDI_RMA_Op_t *curr_ptr; @@ -2470,7 +2484,7 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o /* Process all operations until they are complete */ while (!MPIDI_CH3I_RMA_Ops_isempty(ops_list)) { int nDone = 0; - mpi_errno = rma_list_gc(win_ptr, ops_list, NULL, &nDone); + mpi_errno = rma_list_gc(win_ptr, ops_list, ops_list_tail, NULL, &nDone); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); ntimes++; @@ -2510,6 +2524,7 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o */ static inline int rma_list_gc(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list, + MPIDI_RMA_Ops_list_t * ops_list_tail, MPIDI_RMA_Op_t * last_elm, int *nDone) { int mpi_errno = 0; @@ -2538,7 +2553,7 @@ static inline int rma_list_gc(MPID_Win * win_ptr, } /* --END ERROR HANDLING-- */ MPID_Request_release(curr_ptr->request); - MPIDI_CH3I_RMA_Ops_free_and_next(ops_list, &curr_ptr); + MPIDI_CH3I_RMA_Ops_free_and_next(win_ptr, ops_list, ops_list_tail, &curr_ptr); nVisit++; /* MT: avoid processing unissued operations enqueued by other diff --git a/src/mpid/ch3/src/mpid_finalize.c b/src/mpid/ch3/src/mpid_finalize.c index 5048ee7..1d60f85 100644 --- a/src/mpid/ch3/src/mpid_finalize.c +++ b/src/mpid/ch3/src/mpid_finalize.c @@ -144,6 +144,8 @@ int MPID_Finalize(void) p = pNext; } } + + MPIDI_RMA_finalize(); MPIU_Free(MPIDI_failed_procs_string); diff --git a/src/mpid/ch3/src/mpid_init.c b/src/mpid/ch3/src/mpid_init.c index e0a9a0f..6344f3b 100644 --- a/src/mpid/ch3/src/mpid_init.c +++ b/src/mpid/ch3/src/mpid_init.c @@ -338,6 +338,9 @@ int MPID_Init(int *argc, char ***argv, int requested, int *provided, NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_RMA_init(); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_INIT); return mpi_errno; diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index 139dd76..7ab57a1 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -257,7 +257,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, int mpi_errno = MPI_SUCCESS; int i; MPID_Comm *win_comm_ptr; - MPIU_CHKPMEM_DECL(1); + MPIU_CHKPMEM_DECL(2); MPIDI_STATE_DECL(MPID_STATE_WIN_INIT); MPIDI_FUNC_ENTER(MPID_STATE_WIN_INIT); @@ -304,6 +304,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->epoch_state = MPIDI_EPOCH_NONE; (*win_ptr)->epoch_count = 0; (*win_ptr)->at_rma_ops_list = NULL; + (*win_ptr)->at_rma_ops_list_tail = NULL; (*win_ptr)->shm_allocated = FALSE; /* Initialize the passive target lock state */ @@ -313,6 +314,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, for (i = 0; i < MPIR_Comm_size(win_comm_ptr); i++) { (*win_ptr)->targets[i].rma_ops_list = NULL; + (*win_ptr)->targets[i].rma_ops_list_tail = NULL; (*win_ptr)->targets[i].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; } @@ -325,6 +327,16 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->info_args.alloc_shared_noncontig = 0; (*win_ptr)->info_args.alloc_shm = FALSE; + MPIU_CHKPMEM_MALLOC((*win_ptr)->op_pool_start, struct MPIDI_RMA_Op *, + sizeof(MPIDI_RMA_Op_t) * MPIR_CVAR_CH3_RMA_OP_WIN_POOL_SIZE, mpi_errno, + "RMA op pool"); + (*win_ptr)->op_pool = NULL; + (*win_ptr)->op_pool_tail = NULL; + for (i = 0; i < MPIR_CVAR_CH3_RMA_OP_WIN_POOL_SIZE; i++) { + (*win_ptr)->op_pool_start[i].pool_type = MPIDI_RMA_POOL_WIN; + MPL_LL_APPEND((*win_ptr)->op_pool, (*win_ptr)->op_pool_tail, &((*win_ptr)->op_pool_start[i])); + } + MPID_WIN_FTABLE_SET_DEFAULTS(win_ptr); fn_exit: diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c index 5a7221c..2cb9bd9 100644 --- a/src/mpid/ch3/src/mpidi_rma.c +++ b/src/mpid/ch3/src/mpidi_rma.c @@ -7,6 +7,87 @@ #include "mpidimpl.h" #include "mpidrma.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_CH3_RMA_OP_WIN_POOL_SIZE + category : CH3 + type : int + default : 256 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Size of the window-private RMA operations pool (in number of + operations) that stores information about RMA operations that + could not be issued immediately. Requires a positive value. + + - name : MPIR_CVAR_CH3_RMA_OP_GLOBAL_POOL_SIZE + category : CH3 + type : int + default : 16384 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Size of the Global RMA operations pool (in number of + operations) that stores information about RMA operations that + could not be issued immediatly. Requires a positive value. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + + +struct MPIDI_RMA_Op *global_rma_op_pool = NULL, *global_rma_op_pool_tail = NULL, *global_rma_op_pool_start = NULL; + +#undef FUNCNAME +#define FUNCNAME MPIDI_RMA_init +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_RMA_init(void) +{ + int mpi_errno = MPI_SUCCESS; + int i; + MPIU_CHKPMEM_DECL(1); + + MPIDI_STATE_DECL(MPID_STATE_MPIDI_RMA_INIT); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_RMA_INIT); + + MPIU_CHKPMEM_MALLOC(global_rma_op_pool_start, struct MPIDI_RMA_Op *, + sizeof(struct MPIDI_RMA_Op) * MPIR_CVAR_CH3_RMA_OP_GLOBAL_POOL_SIZE, + mpi_errno, "RMA op pool"); + for (i = 0; i < MPIR_CVAR_CH3_RMA_OP_GLOBAL_POOL_SIZE; i++) { + global_rma_op_pool_start[i].pool_type = MPIDI_RMA_POOL_GLOBAL; + MPL_LL_APPEND(global_rma_op_pool, global_rma_op_pool_tail, &(global_rma_op_pool_start[i])); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RMA_INIT); + return mpi_errno; + + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_fail; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_RMA_finalize +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +void MPIDI_RMA_finalize(void) +{ + MPIDI_STATE_DECL(MPID_STATE_MPIDI_RMA_FINALIZE); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_RMA_FINALIZE); + + MPIU_Free(global_rma_op_pool_start); + + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RMA_FINALIZE); +} + #undef FUNCNAME #define FUNCNAME MPIDI_Win_free @@ -43,6 +124,7 @@ int MPIDI_Win_free(MPID_Win ** win_ptr) MPIU_Free((*win_ptr)->sizes); MPIU_Free((*win_ptr)->disp_units); MPIU_Free((*win_ptr)->all_win_handles); + MPIU_Free((*win_ptr)->op_pool_start); /* Free the attached buffer for windows created with MPI_Win_allocate() */ if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE || http://git.mpich.org/mpich.git/commitdiff/b1685139330580132c64e4b616ae91824… commit b1685139330580132c64e4b616ae91824b9bf5b8 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Nov 2 21:54:13 2014 -0600 Embedding packet structure into RMA operation structure. We were duplicating information in the operation structure and in the packet structure when the message is actually issued. Since most of the information is the same anyway, this patch just embeds a packet structure into the operation structure, so that we eliminate unnessary copy. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpi/errhan/errnames.txt b/src/mpi/errhan/errnames.txt index 060b3b4..4007488 100644 --- a/src/mpi/errhan/errnames.txt +++ b/src/mpi/errhan/errnames.txt @@ -189,6 +189,9 @@ MPI_TYPECLASS_INTEGER, or MPI_TYPECLASS_COMPLEX **memcpyalias:memcpy arguments alias each other **memcpyalias %p %p %L:memcpy arguments alias each other, dst=%p src=%p len=%L +**invalidpkt: Invalid packet type +**invalidpkt %d: Invalid packet type (%d) + **rmatypenotatomic: Datatype not permitted for atomic operations **rmatypenotatomic %D: Datatype (%D) not permitted for atomic operations **winflavor: Window flavor is not compatible with the given operation diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 234c3dc..c95847c 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -9,17 +9,6 @@ #include "mpidi_ch3_impl.h" -typedef enum MPIDI_RMA_Op_type { - MPIDI_RMA_PUT = 23, - MPIDI_RMA_GET = 24, - MPIDI_RMA_ACCUMULATE = 25, - /* REMOVED: MPIDI_RMA_LOCK = 26, */ - MPIDI_RMA_ACC_CONTIG = 27, - MPIDI_RMA_GET_ACCUMULATE = 28, - MPIDI_RMA_COMPARE_AND_SWAP = 29, - MPIDI_RMA_FETCH_AND_OP = 30 -} MPIDI_RMA_Op_type_t; - /* Special case RMA operations */ enum MPIDI_RMA_Datatype { @@ -61,33 +50,29 @@ typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */ typedef struct MPIDI_RMA_Op { struct MPIDI_RMA_Op *prev; /* pointer to next element in list */ struct MPIDI_RMA_Op *next; /* pointer to next element in list */ - /* FIXME: It would be better to setup the packet that will be sent, at - * least in most cases (if, as a result of the sync/ops/sync sequence, - * a different packet type is needed, it can be extracted from the - * information otherwise stored). */ - MPIDI_RMA_Op_type_t type; + void *origin_addr; int origin_count; MPI_Datatype origin_datatype; - int target_rank; - MPI_Aint target_disp; - int target_count; - MPI_Datatype target_datatype; - MPI_Op op; /* for accumulate */ - /* Used to complete operations */ - struct MPID_Request *request; - MPIDI_RMA_dtype_info dtype_info; - void *dataloop; + + void *compare_addr; + MPI_Datatype compare_datatype; + void *result_addr; int result_count; MPI_Datatype result_datatype; - void *compare_addr; - int compare_count; - MPI_Datatype compare_datatype; + + struct MPID_Request *request; + MPIDI_RMA_dtype_info dtype_info; + void *dataloop; + + int target_rank; + + MPIDI_CH3_Pkt_t pkt; } MPIDI_RMA_Op_t; typedef struct MPIDI_PT_single_op { - int type; /* put, get, or accum. */ + MPIDI_CH3_Pkt_type_t type; /* put, get, or accum. */ void *addr; int count; MPI_Datatype datatype; diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index da2b500..caf1f47 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -193,6 +193,43 @@ MPIDI_CH3_PKT_DEFS #endif /* *INDENT-ON* */ +#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_) \ + { \ + err_ = MPI_SUCCESS; \ + switch(pkt_.type) { \ + case (MPIDI_CH3_PKT_PUT): \ + datatype_ = pkt_.put.datatype; \ + break; \ + case (MPIDI_CH3_PKT_GET): \ + datatype_ = pkt_.get.datatype; \ + break; \ + case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_GET_ACCUM): \ + datatype_ = pkt_.accum.datatype; \ + break; \ + case (MPIDI_CH3_PKT_CAS): \ + datatype_ = pkt_.cas.datatype; \ + break; \ + case (MPIDI_CH3_PKT_FOP): \ + datatype_ = pkt_.fop.datatype; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_PUT_UNLOCK): \ + datatype_ = pkt_.lock_put_unlock.datatype; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_GET_UNLOCK): \ + datatype_ = pkt_.lock_get_unlock.datatype; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK): \ + datatype_ = pkt_.lock_accum_unlock.datatype; \ + break; \ + case (MPIDI_CH3_PKT_ACCUM_IMMED): \ + datatype_ = pkt_.accum_immed.datatype; \ + break; \ + default: \ + MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", pkt_.type); \ + } \ + } + typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; @@ -279,6 +316,7 @@ typedef struct MPIDI_CH3_Pkt_cas { MPI_Datatype datatype; void *addr; MPI_Request request_handle; + MPI_Win source_win_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -300,6 +338,7 @@ typedef struct MPIDI_CH3_Pkt_fop { void *addr; MPI_Op op; MPI_Request request_handle; + MPI_Win source_win_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index fd66526..00664ba 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -12,8 +12,7 @@ #include "mpid_rma_shm.h" int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle, - MPI_Win target_win_handle); + MPIDI_CH3_Pkt_flags_t flags); #undef FUNCNAME #define FUNCNAME send_lock_msg diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 5a8de15..d82fb53 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -905,7 +905,7 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) MPIDI_PT_single_op * single_op; single_op = lock_queue->pt_single_op; - if (single_op->type == MPIDI_RMA_PUT) { + if (single_op->type == MPIDI_CH3_PKT_LOCK_PUT_UNLOCK) { mpi_errno = MPIR_Localcopy(single_op->data, single_op->count, single_op->datatype, @@ -913,21 +913,21 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) single_op->count, single_op->datatype); } - else if (single_op->type == MPIDI_RMA_ACCUMULATE) { + else if (single_op->type == MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK) { if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); mpi_errno = do_simple_accumulate(single_op); if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); } - else if (single_op->type == MPIDI_RMA_GET) { + else if (single_op->type == MPIDI_CH3_PKT_LOCK_GET_UNLOCK) { mpi_errno = do_simple_get(win_ptr, lock_queue); } if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } /* if put or accumulate, send rma done packet and release lock. */ - if (single_op->type != MPIDI_RMA_GET) { + if (single_op->type != MPIDI_CH3_PKT_LOCK_GET_UNLOCK) { /* NOTE: Only *queued* single_op operations are completed here. Lock-op-unlock/single_op RMA ops can also be completed as they arrive within various packet/request handlers via diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c index b8b113f..ad9307d 100644 --- a/src/mpid/ch3/src/ch3u_rma_oplist.c +++ b/src/mpid/ch3/src/ch3u_rma_oplist.c @@ -110,15 +110,10 @@ static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, #define FUNCNAME send_rma_msg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, - MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request) +static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_put_t *put_pkt = &upkt.put; - MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum; + MPIDI_CH3_Pkt_put_t *put_pkt = &rma_op->pkt.put; + MPIDI_CH3_Pkt_accum_t *accum_pkt = &rma_op->pkt.accum; MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; int origin_dt_derived, target_dt_derived, iovcnt; @@ -126,6 +121,7 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_VC_t *vc; MPID_Comm *comm_ptr; MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL; + MPI_Datatype target_datatype; MPID_Request *resp_req = NULL; MPIU_CHKPMEM_DECL(1); MPIDI_STATE_DECL(MPID_STATE_SEND_RMA_MSG); @@ -133,23 +129,15 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_RMA_MSG); - *request = NULL; + rma_op->request = NULL; - if (rma_op->type == MPIDI_RMA_PUT) { - MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); - put_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT) { put_pkt->flags = flags; - put_pkt->count = rma_op->target_count; - put_pkt->datatype = rma_op->target_datatype; - put_pkt->dataloop_size = 0; - put_pkt->target_win_handle = target_win_handle; - put_pkt->source_win_handle = source_win_handle; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt; iov[0].MPID_IOV_LEN = sizeof(*put_pkt); } - else if (rma_op->type == MPIDI_RMA_GET_ACCUMULATE) { + else if (rma_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) { /* Create a request for the GACC response. Store the response buf, count, and * datatype in it, and pass the request's handle in the GACC packet. When the * response comes from the target, it will contain the request handle. */ @@ -161,8 +149,8 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, resp_req->dev.user_buf = rma_op->result_addr; resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = target_win_handle; - resp_req->dev.source_win_handle = source_win_handle; + resp_req->dev.target_win_handle = accum_pkt->target_win_handle; + resp_req->dev.source_win_handle = accum_pkt->source_win_handle; if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { MPID_Datatype *result_dtp = NULL; @@ -173,32 +161,14 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, } /* Note: Get_accumulate uses the same packet type as accumulate */ - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); - accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - accum_pkt->flags = flags; - accum_pkt->count = rma_op->target_count; - accum_pkt->datatype = rma_op->target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = rma_op->op; - accum_pkt->target_win_handle = target_win_handle; - accum_pkt->source_win_handle = source_win_handle; accum_pkt->request_handle = resp_req->handle; + accum_pkt->flags = flags; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); } else { - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); - accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; accum_pkt->flags = flags; - accum_pkt->count = rma_op->target_count; - accum_pkt->datatype = rma_op->target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = rma_op->op; - accum_pkt->target_win_handle = target_win_handle; - accum_pkt->source_win_handle = source_win_handle; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); @@ -220,9 +190,10 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, origin_dt_derived = 0; } - if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) { + MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(rma_op->pkt, target_datatype, mpi_errno); + if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { target_dt_derived = 1; - MPID_Datatype_get_ptr(rma_op->target_datatype, target_dtp); + MPID_Datatype_get_ptr(target_datatype, target_dtp); } else { target_dt_derived = 0; @@ -230,31 +201,32 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, if (target_dt_derived) { /* derived datatype on target. fill derived datatype info */ - dtype_info->is_contig = target_dtp->is_contig; - dtype_info->max_contig_blocks = target_dtp->max_contig_blocks; - dtype_info->size = target_dtp->size; - dtype_info->extent = target_dtp->extent; - dtype_info->dataloop_size = target_dtp->dataloop_size; - dtype_info->dataloop_depth = target_dtp->dataloop_depth; - dtype_info->eltype = target_dtp->eltype; - dtype_info->dataloop = target_dtp->dataloop; - dtype_info->ub = target_dtp->ub; - dtype_info->lb = target_dtp->lb; - dtype_info->true_ub = target_dtp->true_ub; - dtype_info->true_lb = target_dtp->true_lb; - dtype_info->has_sticky_ub = target_dtp->has_sticky_ub; - dtype_info->has_sticky_lb = target_dtp->has_sticky_lb; - - MPIU_CHKPMEM_MALLOC(*dataloop, void *, target_dtp->dataloop_size, mpi_errno, "dataloop"); + rma_op->dtype_info.is_contig = target_dtp->is_contig; + rma_op->dtype_info.max_contig_blocks = target_dtp->max_contig_blocks; + rma_op->dtype_info.size = target_dtp->size; + rma_op->dtype_info.extent = target_dtp->extent; + rma_op->dtype_info.dataloop_size = target_dtp->dataloop_size; + rma_op->dtype_info.dataloop_depth = target_dtp->dataloop_depth; + rma_op->dtype_info.eltype = target_dtp->eltype; + rma_op->dtype_info.dataloop = target_dtp->dataloop; + rma_op->dtype_info.ub = target_dtp->ub; + rma_op->dtype_info.lb = target_dtp->lb; + rma_op->dtype_info.true_ub = target_dtp->true_ub; + rma_op->dtype_info.true_lb = target_dtp->true_lb; + rma_op->dtype_info.has_sticky_ub = target_dtp->has_sticky_ub; + rma_op->dtype_info.has_sticky_lb = target_dtp->has_sticky_lb; + + MPIU_CHKPMEM_MALLOC(rma_op->dataloop, void *, target_dtp->dataloop_size, + mpi_errno, "dataloop"); MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY); - MPIU_Memcpy(*dataloop, target_dtp->dataloop, target_dtp->dataloop_size); + MPIU_Memcpy(rma_op->dataloop, target_dtp->dataloop, target_dtp->dataloop_size); MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY); /* the dataloop can have undefined padding sections, so we need to let * valgrind know that it is OK to pass this data to writev later on */ - MPL_VG_MAKE_MEM_DEFINED(*dataloop, target_dtp->dataloop_size); + MPL_VG_MAKE_MEM_DEFINED(rma_op->dataloop, target_dtp->dataloop_size); - if (rma_op->type == MPIDI_RMA_PUT) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT) { put_pkt->dataloop_size = target_dtp->dataloop_size; } else { @@ -272,36 +244,36 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; iovcnt = 2; MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request); + mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, &rma_op->request); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); } else { /* derived datatype on origin */ - *request = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(*request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + rma_op->request = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(rma_op->request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - MPIU_Object_set_ref(*request, 2); - (*request)->kind = MPID_REQUEST_SEND; + MPIU_Object_set_ref(rma_op->request, 2); + rma_op->request->kind = MPID_REQUEST_SEND; - (*request)->dev.segment_ptr = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, + rma_op->request->dev.segment_ptr = MPID_Segment_alloc(); + MPIU_ERR_CHKANDJUMP1(rma_op->request->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc"); - (*request)->dev.datatype_ptr = origin_dtp; + rma_op->request->dev.datatype_ptr = origin_dtp; /* this will cause the datatype to be freed when the request * is freed. */ MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count, - rma_op->origin_datatype, (*request)->dev.segment_ptr, 0); - (*request)->dev.segment_first = 0; - (*request)->dev.segment_size = rma_op->origin_count * origin_type_size; + rma_op->origin_datatype, rma_op->request->dev.segment_ptr, 0); + rma_op->request->dev.segment_first = 0; + rma_op->request->dev.segment_size = rma_op->origin_count * origin_type_size; - (*request)->dev.OnFinal = 0; - (*request)->dev.OnDataAvail = 0; + rma_op->request->dev.OnFinal = 0; + rma_op->request->dev.OnDataAvail = 0; MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = - vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); + vc->sendNoncontig_fn(vc, rma_op->request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); } @@ -310,38 +282,40 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, /* derived datatype on target */ MPID_Datatype *combined_dtp = NULL; - *request = MPID_Request_create(); - if (*request == NULL) { + rma_op->request = MPID_Request_create(); + if (rma_op->request == NULL) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); } - MPIU_Object_set_ref(*request, 2); - (*request)->kind = MPID_REQUEST_SEND; + MPIU_Object_set_ref(rma_op->request, 2); + rma_op->request->kind = MPID_REQUEST_SEND; - (*request)->dev.segment_ptr = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, + rma_op->request->dev.segment_ptr = MPID_Segment_alloc(); + MPIU_ERR_CHKANDJUMP1(rma_op->request->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc"); /* create a new datatype containing the dtype_info, dataloop, and origin data */ mpi_errno = - create_datatype(dtype_info, *dataloop, target_dtp->dataloop_size, rma_op->origin_addr, - rma_op->origin_count, rma_op->origin_datatype, &combined_dtp); + create_datatype(&rma_op->dtype_info, rma_op->dataloop, target_dtp->dataloop_size, + rma_op->origin_addr, rma_op->origin_count, rma_op->origin_datatype, + &combined_dtp); if (mpi_errno) MPIU_ERR_POP(mpi_errno); - (*request)->dev.datatype_ptr = combined_dtp; + rma_op->request->dev.datatype_ptr = combined_dtp; /* combined_datatype will be freed when request is freed */ - MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, (*request)->dev.segment_ptr, 0); - (*request)->dev.segment_first = 0; - (*request)->dev.segment_size = combined_dtp->size; + MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, rma_op->request->dev.segment_ptr, 0); + rma_op->request->dev.segment_first = 0; + rma_op->request->dev.segment_size = combined_dtp->size; - (*request)->dev.OnFinal = 0; - (*request)->dev.OnDataAvail = 0; + rma_op->request->dev.OnFinal = 0; + rma_op->request->dev.OnDataAvail = 0; MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); + mpi_errno = + vc->sendNoncontig_fn(vc, rma_op->request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -354,7 +328,7 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, /* This operation can generate two requests; one for inbound and one for * outbound data. */ if (resp_req != NULL) { - if (*request != NULL) { + if (rma_op->request != NULL) { /* If we have both inbound and outbound requests (i.e. GACC * operation), we need to ensure that the source buffer is * available and that the response data has been received before @@ -370,12 +344,12 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, * it will be completed by the progress engine. */ - MPID_Request_release(*request); - *request = resp_req; + MPID_Request_release(rma_op->request); + rma_op->request = resp_req; } else { - *request = resp_req; + rma_op->request = resp_req; } /* For error checking */ @@ -391,13 +365,13 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, if (resp_req) { MPID_Request_release(resp_req); } - if (*request) { + if (rma_op->request) { MPIU_CHKPMEM_REAP(); - if ((*request)->dev.datatype_ptr) - MPID_Datatype_release((*request)->dev.datatype_ptr); - MPID_Request_release(*request); + if (rma_op->request->dev.datatype_ptr) + MPID_Datatype_release(rma_op->request->dev.datatype_ptr); + MPID_Request_release(rma_op->request); } - *request = NULL; + rma_op->request = NULL; goto fn_exit; /* --END ERROR HANDLING-- */ } @@ -410,13 +384,9 @@ static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, - MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, MPID_Request ** request) + MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum; + MPIDI_CH3_Pkt_accum_t *accum_pkt = &rma_op->pkt.accum; MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; int iovcnt; @@ -428,24 +398,16 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_CONTIG_ACC_MSG); - *request = NULL; + rma_op->request = NULL; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); /* FIXME: Make this size check efficient and match the packet type */ MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) { - MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &upkt.accum_immed; + MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &rma_op->pkt.accum_immed; void *dest = accumi_pkt->data, *src = rma_op->origin_addr; - MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED); - accumi_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; accumi_pkt->flags = flags; - accumi_pkt->count = rma_op->target_count; - accumi_pkt->datatype = rma_op->target_datatype; - accumi_pkt->op = rma_op->op; - accumi_pkt->target_win_handle = target_win_handle; - accumi_pkt->source_win_handle = source_win_handle; switch (len) { case 1: @@ -466,22 +428,13 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, comm_ptr = win_ptr->comm_ptr; MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), request); + mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), &rma_op->request); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); goto fn_exit; } - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); - accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; accum_pkt->flags = flags; - accum_pkt->count = rma_op->target_count; - accum_pkt->datatype = rma_op->target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = rma_op->op; - accum_pkt->target_win_handle = target_win_handle; - accum_pkt->source_win_handle = source_win_handle; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); @@ -505,7 +458,7 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; iovcnt = 2; MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request); + mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, &rma_op->request); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -514,10 +467,10 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: - if (*request) { - MPID_Request_release(*request); + if (rma_op->request) { + MPID_Request_release(rma_op->request); } - *request = NULL; + rma_op->request = NULL; goto fn_exit; /* --END ERROR HANDLING-- */ } @@ -531,13 +484,10 @@ static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, - MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, MPID_Request ** request) + MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags) { int mpi_errno = MPI_SUCCESS; - MPID_Request *rmw_req = NULL, *resp_req = NULL; + MPID_Request *rmw_req = NULL; MPIDI_VC_t *vc; MPID_Comm *comm_ptr; MPI_Aint len; @@ -545,44 +495,35 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_IMMED_RMW_MSG); - *request = NULL; - /* Create a request for the RMW response. Store the origin buf, count, and * datatype in it, and pass the request's handle RMW packet. When the * response comes from the target, it will contain the request handle. */ - resp_req = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - *request = resp_req; + rma_op->request = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(rma_op->request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); /* Set refs on the request to 2: one for the response message, and one for * the partial completion handler */ - MPIU_Object_set_ref(resp_req, 2); + MPIU_Object_set_ref(rma_op->request, 2); - resp_req->dev.user_buf = rma_op->result_addr; - resp_req->dev.user_count = rma_op->result_count; - resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = target_win_handle; - resp_req->dev.source_win_handle = source_win_handle; + rma_op->request->dev.user_buf = rma_op->result_addr; + rma_op->request->dev.user_count = rma_op->result_count; + rma_op->request->dev.datatype = rma_op->result_datatype; /* REQUIRE: All datatype arguments must be of the same, builtin * type and counts must be 1. */ MPID_Datatype_get_size_macro(rma_op->origin_datatype, len); comm_ptr = win_ptr->comm_ptr; - if (rma_op->type == MPIDI_RMA_COMPARE_AND_SWAP) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_cas_t *cas_pkt = &upkt.cas; + if (rma_op->pkt.type == MPIDI_CH3_PKT_CAS) { + MPIDI_CH3_Pkt_cas_t *cas_pkt = &rma_op->pkt.cas; MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); - MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; + rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle; - cas_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags = flags; - cas_pkt->datatype = rma_op->target_datatype; - cas_pkt->target_win_handle = target_win_handle; - cas_pkt->request_handle = resp_req->handle; MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); @@ -598,25 +539,20 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, } } - else if (rma_op->type == MPIDI_RMA_FETCH_AND_OP) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_fop_t *fop_pkt = &upkt.fop; + else if (rma_op->pkt.type == MPIDI_CH3_PKT_FOP) { + MPIDI_CH3_Pkt_fop_t *fop_pkt = &rma_op->pkt.fop; MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u)); - MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + rma_op->request->dev.target_win_handle = fop_pkt->target_win_handle; + rma_op->request->dev.source_win_handle = fop_pkt->source_win_handle; - fop_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + fop_pkt->request_handle = rma_op->request->handle; fop_pkt->flags = flags; - fop_pkt->datatype = rma_op->target_datatype; - fop_pkt->target_win_handle = target_win_handle; - fop_pkt->request_handle = resp_req->handle; - fop_pkt->op = rma_op->op; - if (len <= sizeof(fop_pkt->origin_data) || rma_op->op == MPI_NO_OP) { + if (len <= sizeof(fop_pkt->origin_data) || fop_pkt->op == MPI_NO_OP) { /* Embed FOP data in the packet header */ - if (rma_op->op != MPI_NO_OP) { + if (fop_pkt->op != MPI_NO_OP) { MPIU_Memcpy(fop_pkt->origin_data, rma_op->origin_addr, len); } @@ -663,10 +599,10 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: - if (*request) { - MPID_Request_release(*request); + if (rma_op->request) { + MPID_Request_release(rma_op->request); } - *request = NULL; + rma_op->request = NULL; if (rmw_req) { MPID_Request_release(rmw_req); } @@ -680,19 +616,15 @@ static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, #define FUNCNAME recv_rma_msg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, - MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request) +static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_get_t *get_pkt = &upkt.get; + MPIDI_CH3_Pkt_get_t *get_pkt = &rma_op->pkt.get; int mpi_errno = MPI_SUCCESS; MPIDI_VC_t *vc; MPID_Comm *comm_ptr; - MPID_Request *req = NULL; MPID_Datatype *dtp; + MPI_Datatype target_datatype; + MPID_Request *req = NULL; MPID_IOV iov[MPID_IOV_LIMIT]; MPIU_CHKPMEM_DECL(1); MPIDI_STATE_DECL(MPID_STATE_RECV_RMA_MSG); @@ -704,36 +636,27 @@ static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, * and pass a handle to it in the get packet. When the get * response comes from the target, it will contain the request * handle. */ - req = MPID_Request_create(); - if (req == NULL) { + rma_op->request = MPID_Request_create(); + if (rma_op->request == NULL) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); } - *request = req; + MPIU_Object_set_ref(rma_op->request, 2); - MPIU_Object_set_ref(req, 2); - - req->dev.user_buf = rma_op->origin_addr; - req->dev.user_count = rma_op->origin_count; - req->dev.datatype = rma_op->origin_datatype; - req->dev.target_win_handle = MPI_WIN_NULL; - req->dev.source_win_handle = source_win_handle; - if (!MPIR_DATATYPE_IS_PREDEFINED(req->dev.datatype)) { - MPID_Datatype_get_ptr(req->dev.datatype, dtp); - req->dev.datatype_ptr = dtp; + rma_op->request->dev.user_buf = rma_op->origin_addr; + rma_op->request->dev.user_count = rma_op->origin_count; + rma_op->request->dev.datatype = rma_op->origin_datatype; + rma_op->request->dev.target_win_handle = MPI_WIN_NULL; + rma_op->request->dev.source_win_handle = get_pkt->source_win_handle; + if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->request->dev.datatype)) { + MPID_Datatype_get_ptr(rma_op->request->dev.datatype, dtp); + rma_op->request->dev.datatype_ptr = dtp; /* this will cause the datatype to be freed when the * request is freed. */ } - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + get_pkt->request_handle = rma_op->request->handle; get_pkt->flags = flags; - get_pkt->count = rma_op->target_count; - get_pkt->datatype = rma_op->target_datatype; - get_pkt->request_handle = req->handle; - get_pkt->target_win_handle = target_win_handle; - get_pkt->source_win_handle = source_win_handle; /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); @@ -743,7 +666,8 @@ static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, comm_ptr = win_ptr->comm_ptr; MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) { + MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(rma_op->pkt, target_datatype, mpi_errno); + if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* basic datatype on target. simply send the get_pkt. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, get_pkt, sizeof(*get_pkt), &req); @@ -753,39 +677,39 @@ static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, /* derived datatype on target. fill derived datatype info and * send it along with get_pkt. */ - MPID_Datatype_get_ptr(rma_op->target_datatype, dtp); - dtype_info->is_contig = dtp->is_contig; - dtype_info->max_contig_blocks = dtp->max_contig_blocks; - dtype_info->size = dtp->size; - dtype_info->extent = dtp->extent; - dtype_info->dataloop_size = dtp->dataloop_size; - dtype_info->dataloop_depth = dtp->dataloop_depth; - dtype_info->eltype = dtp->eltype; - dtype_info->dataloop = dtp->dataloop; - dtype_info->ub = dtp->ub; - dtype_info->lb = dtp->lb; - dtype_info->true_ub = dtp->true_ub; - dtype_info->true_lb = dtp->true_lb; - dtype_info->has_sticky_ub = dtp->has_sticky_ub; - dtype_info->has_sticky_lb = dtp->has_sticky_lb; - - MPIU_CHKPMEM_MALLOC(*dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop"); + MPID_Datatype_get_ptr(target_datatype, dtp); + rma_op->dtype_info.is_contig = dtp->is_contig; + rma_op->dtype_info.max_contig_blocks = dtp->max_contig_blocks; + rma_op->dtype_info.size = dtp->size; + rma_op->dtype_info.extent = dtp->extent; + rma_op->dtype_info.dataloop_size = dtp->dataloop_size; + rma_op->dtype_info.dataloop_depth = dtp->dataloop_depth; + rma_op->dtype_info.eltype = dtp->eltype; + rma_op->dtype_info.dataloop = dtp->dataloop; + rma_op->dtype_info.ub = dtp->ub; + rma_op->dtype_info.lb = dtp->lb; + rma_op->dtype_info.true_ub = dtp->true_ub; + rma_op->dtype_info.true_lb = dtp->true_lb; + rma_op->dtype_info.has_sticky_ub = dtp->has_sticky_ub; + rma_op->dtype_info.has_sticky_lb = dtp->has_sticky_lb; + + MPIU_CHKPMEM_MALLOC(rma_op->dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop"); MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY); - MPIU_Memcpy(*dataloop, dtp->dataloop, dtp->dataloop_size); + MPIU_Memcpy(rma_op->dataloop, dtp->dataloop, dtp->dataloop_size); MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY); /* the dataloop can have undefined padding sections, so we need to let * valgrind know that it is OK to pass this data to writev later on */ - MPL_VG_MAKE_MEM_DEFINED(*dataloop, dtp->dataloop_size); + MPL_VG_MAKE_MEM_DEFINED(rma_op->dataloop, dtp->dataloop_size); get_pkt->dataloop_size = dtp->dataloop_size; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) dtype_info; - iov[1].MPID_IOV_LEN = sizeof(*dtype_info); - iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) * dataloop; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & rma_op->dtype_info; + iov[1].MPID_IOV_LEN = sizeof(rma_op->dtype_info); + iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->dataloop; iov[2].MPID_IOV_LEN = dtp->dataloop_size; MPIU_THREAD_CS_ENTER(CH3COMM, vc); @@ -858,70 +782,36 @@ static int send_flush_msg(int dest, MPID_Win * win_ptr) int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle, - MPI_Win target_win_handle) + MPIDI_CH3_Pkt_flags_t flags) { int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP); - switch (op_ptr->type) { - case (MPIDI_RMA_PUT): - case (MPIDI_RMA_ACCUMULATE): - mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle, - target_win_handle, &op_ptr->dtype_info, - &op_ptr->dataloop, &op_ptr->request); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); + switch (op_ptr->pkt.type) { + case (MPIDI_CH3_PKT_PUT): + case (MPIDI_CH3_PKT_ACCUMULATE): + case (MPIDI_CH3_PKT_GET_ACCUM): + mpi_errno = send_rma_msg(op_ptr, win_ptr, flags); break; - case (MPIDI_RMA_GET_ACCUMULATE): - if (op_ptr->op == MPI_NO_OP) { - /* Note: Origin arguments are ignored for NO_OP, so we don't - * need to release a ref to the origin datatype. */ - - /* Convert the GAcc to a Get */ - op_ptr->type = MPIDI_RMA_GET; - op_ptr->origin_addr = op_ptr->result_addr; - op_ptr->origin_count = op_ptr->result_count; - op_ptr->origin_datatype = op_ptr->result_datatype; - - mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags, source_win_handle, - target_win_handle, &op_ptr->dtype_info, - &op_ptr->dataloop, &op_ptr->request); - } - else { - mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle, - target_win_handle, &op_ptr->dtype_info, - &op_ptr->dataloop, &op_ptr->request); - } - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); + case (MPIDI_CH3_PKT_ACCUM_IMMED): + mpi_errno = send_contig_acc_msg(op_ptr, win_ptr, flags); break; - case (MPIDI_RMA_ACC_CONTIG): - mpi_errno = send_contig_acc_msg(op_ptr, win_ptr, flags, - source_win_handle, target_win_handle, &op_ptr->request); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); + case (MPIDI_CH3_PKT_GET): + mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags); break; - case (MPIDI_RMA_GET): - mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags, - source_win_handle, target_win_handle, - &op_ptr->dtype_info, &op_ptr->dataloop, &op_ptr->request); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - break; - case (MPIDI_RMA_COMPARE_AND_SWAP): - case (MPIDI_RMA_FETCH_AND_OP): - mpi_errno = send_immed_rmw_msg(op_ptr, win_ptr, flags, - source_win_handle, target_win_handle, &op_ptr->request); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); + case (MPIDI_CH3_PKT_CAS): + case (MPIDI_CH3_PKT_FOP): + mpi_errno = send_immed_rmw_msg(op_ptr, win_ptr, flags); break; default: MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winInvalidOp"); } + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP); return mpi_errno; diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 9024726..f45a2b4 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -74,6 +74,7 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_CH3_Pkt_put_t *put_pkt = NULL; /* queue it up */ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); @@ -81,18 +82,21 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIU_ERR_POP(mpi_errno); } + put_pkt = &(new_ptr->pkt.put); + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + put_pkt->count = target_count; + put_pkt->datatype = target_datatype; + put_pkt->dataloop_size = 0; + put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + put_pkt->source_win_handle = win_ptr->handle; + /* FIXME: For contig and very short operations, use a streamlined op */ - new_ptr->type = MPIDI_RMA_PUT; - /* Cast away const'ness for the origin address, as the - * MPIDI_RMA_Op_t structure is used for both PUT and GET like - * operations */ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = origin_count; new_ptr->origin_datatype = origin_datatype; new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = target_count; - new_ptr->target_datatype = target_datatype; /* if source or target datatypes are derived, increment their * reference counts */ @@ -181,6 +185,7 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_CH3_Pkt_get_t *get_pkt = NULL; /* queue it up */ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); @@ -188,15 +193,21 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype MPIU_ERR_POP(mpi_errno); } + get_pkt = &(new_ptr->pkt.get); + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_pkt->count = target_count; + get_pkt->datatype = target_datatype; + get_pkt->dataloop_size = 0; + get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_pkt->source_win_handle = win_ptr->handle; + /* FIXME: For contig and very short operations, use a streamlined op */ - new_ptr->type = MPIDI_RMA_GET; new_ptr->origin_addr = origin_addr; new_ptr->origin_count = origin_count; new_ptr->origin_datatype = origin_datatype; new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = target_count; - new_ptr->target_datatype = target_datatype; /* if source or target datatypes are derived, increment their * reference counts */ @@ -286,6 +297,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; /* queue it up */ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); @@ -296,32 +308,48 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype /* If predefined and contiguous, use a simplified element */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && enableShortACC) { - new_ptr->type = MPIDI_RMA_ACC_CONTIG; - /* Only the information needed for the contig/predefined acc */ - /* Cast away const'ness for origin_address as - * MPIDI_RMA_Op_t contain both PUT and GET like ops */ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = origin_count; - new_ptr->origin_datatype = origin_datatype; - new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = target_count; - new_ptr->target_datatype = target_datatype; - new_ptr->op = op; - goto fn_exit; + MPI_Aint origin_type_size; + size_t len; + + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); + if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) { + MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt; + + accumi_pkt = &(new_ptr->pkt.accum_immed); + MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED); + accumi_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + accumi_pkt->count = target_count; + accumi_pkt->datatype = target_datatype; + accumi_pkt->op = op; + accumi_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + accumi_pkt->source_win_handle = win_ptr->handle; + + new_ptr->origin_addr = (void *) origin_addr; + new_ptr->origin_count = origin_count; + new_ptr->origin_datatype = origin_datatype; + new_ptr->target_rank = target_rank; + goto fn_exit; + } } - new_ptr->type = MPIDI_RMA_ACCUMULATE; - /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t - * contain both PUT and GET like ops */ + accum_pkt = &(new_ptr->pkt.accum); + + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + accum_pkt->count = target_count; + accum_pkt->datatype = target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = op; + accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + accum_pkt->source_win_handle = win_ptr->handle; + new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = origin_count; new_ptr->origin_datatype = origin_datatype; new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = target_count; - new_ptr->target_datatype = target_datatype; - new_ptr->op = op; /* if source or target datatypes are derived, increment their * reference counts */ @@ -423,20 +451,44 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */ - new_ptr->type = MPIDI_RMA_GET_ACCUMULATE; - /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t - * contain both PUT and GET like ops */ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = origin_count; - new_ptr->origin_datatype = origin_datatype; - new_ptr->result_addr = result_addr; - new_ptr->result_count = result_count; - new_ptr->result_datatype = result_datatype; - new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = target_count; - new_ptr->target_datatype = target_datatype; - new_ptr->op = op; + if (op == MPI_NO_OP) { + /* Convert GAcc to a Get */ + MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get); + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_pkt->count = target_count; + get_pkt->datatype = target_datatype; + get_pkt->dataloop_size = 0; + get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_pkt->source_win_handle = win_ptr->handle; + + new_ptr->origin_addr = result_addr; + new_ptr->origin_count = result_count; + new_ptr->origin_datatype = result_datatype; + new_ptr->target_rank = target_rank; + } + + else { + MPIDI_CH3_Pkt_accum_t *accum_pkt = &(new_ptr->pkt.accum); + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + accum_pkt->count = target_count; + accum_pkt->datatype = target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = op; + accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + accum_pkt->source_win_handle = win_ptr->handle; + + new_ptr->origin_addr = (void *) origin_addr; + new_ptr->origin_count = origin_count; + new_ptr->origin_datatype = origin_datatype; + new_ptr->result_addr = result_addr; + new_ptr->result_count = result_count; + new_ptr->result_datatype = result_datatype; + new_ptr->target_rank = target_rank; + } /* if source or target datatypes are derived, increment their * reference counts */ @@ -524,6 +576,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL; /* Append this operation to the RMA ops queue */ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); @@ -531,20 +584,22 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIU_ERR_POP(mpi_errno); } - new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP; + cas_pkt = &(new_ptr->pkt.cas); + MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + cas_pkt->datatype = datatype; + cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + cas_pkt->source_win_handle = win_ptr->handle; + new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; new_ptr->origin_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = 1; - new_ptr->target_datatype = datatype; new_ptr->result_addr = result_addr; - new_ptr->result_count = 1; new_ptr->result_datatype = datatype; new_ptr->compare_addr = (void *) compare_addr; - new_ptr->compare_count = 1; new_ptr->compare_datatype = datatype; + new_ptr->target_rank = target_rank; } fn_exit: @@ -615,6 +670,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_CH3_Pkt_fop_t *fop_pkt = NULL; /* Append this operation to the RMA ops queue */ mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); @@ -622,18 +678,21 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIU_ERR_POP(mpi_errno); } - new_ptr->type = MPIDI_RMA_FETCH_AND_OP; + fop_pkt = &(new_ptr->pkt.fop); + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + fop_pkt->datatype = datatype; + fop_pkt->op = op; + fop_pkt->source_win_handle = win_ptr->handle; + fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; new_ptr->origin_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->target_disp = target_disp; - new_ptr->target_count = 1; - new_ptr->target_datatype = datatype; new_ptr->result_addr = result_addr; - new_ptr->result_count = 1; new_ptr->result_datatype = datatype; - new_ptr->op = op; + new_ptr->target_rank = target_rank; } fn_exit: diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 9e88165..a0df5bb 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -1003,7 +1003,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle; new_ptr->origin_rank = lock_put_unlock_pkt->origin_rank; - new_ptr->pt_single_op->type = MPIDI_RMA_PUT; + new_ptr->pt_single_op->type = MPIDI_CH3_PKT_LOCK_PUT_UNLOCK; new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags; new_ptr->pt_single_op->addr = lock_put_unlock_pkt->addr; new_ptr->pt_single_op->count = lock_put_unlock_pkt->count; @@ -1151,7 +1151,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle; new_ptr->origin_rank = lock_get_unlock_pkt->origin_rank; - new_ptr->pt_single_op->type = MPIDI_RMA_GET; + new_ptr->pt_single_op->type = MPIDI_CH3_PKT_LOCK_GET_UNLOCK; new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags; new_ptr->pt_single_op->addr = lock_get_unlock_pkt->addr; new_ptr->pt_single_op->count = lock_get_unlock_pkt->count; @@ -1242,7 +1242,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle; new_ptr->origin_rank = lock_accum_unlock_pkt->origin_rank; - new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE; + new_ptr->pt_single_op->type = MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK; new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags; new_ptr->pt_single_op->addr = lock_accum_unlock_pkt->addr; new_ptr->pt_single_op->count = lock_accum_unlock_pkt->count; diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index fde0a90..3629dd5 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -367,7 +367,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) MPIDI_RMA_Op_t *curr_ptr; MPIDI_RMA_Ops_list_t *ops_list; MPID_Comm *comm_ptr; - MPI_Win source_win_handle, target_win_handle; MPID_Progress_state progress_state; int errflag = FALSE; MPIU_CHKLMEM_DECL(3); @@ -487,11 +486,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) flags = MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE; } - source_win_handle = win_ptr->handle; - target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; - - mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, - source_win_handle, target_win_handle); + mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -933,7 +928,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) MPIDI_RMA_Op_t *curr_ptr; MPIDI_RMA_Ops_list_t *ops_list; MPID_Comm *comm_ptr; - MPI_Win source_win_handle, target_win_handle; int start_grp_size, *ranks_in_win_grp, rank; int nRequest = 0; int nRequestNew = 0; @@ -1024,11 +1018,7 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) flags = MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE; } - source_win_handle = win_ptr->handle; - target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; - - mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, - source_win_handle, target_win_handle); + mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -1410,8 +1400,8 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) if (MPIR_CVAR_CH3_RMA_MERGE_LOCK_OP_UNLOCK && win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED && rma_op && rma_op->next == NULL /* There is only one op */ && - rma_op->type != MPIDI_RMA_COMPARE_AND_SWAP && - rma_op->type != MPIDI_RMA_FETCH_AND_OP && rma_op->type != MPIDI_RMA_GET_ACCUMULATE) { + rma_op->pkt.type != MPIDI_CH3_PKT_CAS && + rma_op->pkt.type != MPIDI_CH3_PKT_FOP && rma_op->pkt.type != MPIDI_CH3_PKT_GET_ACCUM) { /* Single put, get, or accumulate between the lock and unlock. If it * is of small size and predefined datatype at the target, we * do an optimization where the lock and the RMA operation are @@ -1420,18 +1410,20 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) MPI_Aint type_size; MPIDI_VC_t *vc; MPIDI_RMA_Op_t *curr_op = rma_op; + MPI_Datatype target_datatype; MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); MPID_Datatype_get_size_macro(curr_op->origin_datatype, type_size); /* msg_sz typically = 65480 */ - if (MPIR_DATATYPE_IS_PREDEFINED(curr_op->target_datatype) && + MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno); + if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && (type_size * curr_op->origin_count <= vc->eager_max_msg_sz)) { single_op_opt = 1; /* Set the lock granted flag to 1 */ win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; - if (curr_op->type == MPIDI_RMA_GET) { + if (curr_op->pkt.type == MPIDI_CH3_PKT_GET) { mpi_errno = send_lock_get(win_ptr, dest); wait_for_rma_done_pkt = 0; } @@ -1912,7 +1904,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, { int mpi_errno = MPI_SUCCESS, nops; MPIDI_RMA_Op_t *curr_ptr; - MPI_Win source_win_handle = MPI_WIN_NULL, target_win_handle = MPI_WIN_NULL; int nRequest = 0, nRequestNew = 0; MPIDI_STATE_DECL(MPID_STATE_DO_PASSIVE_TARGET_RMA); @@ -1935,9 +1926,9 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, /* Check if we can piggyback the RMA done acknowlegdement on the last * operation in the epoch. */ - if (tail->type == MPIDI_RMA_GET || - tail->type == MPIDI_RMA_COMPARE_AND_SWAP || - tail->type == MPIDI_RMA_FETCH_AND_OP || tail->type == MPIDI_RMA_GET_ACCUMULATE) { + if (tail->pkt.type == MPIDI_CH3_PKT_GET || + tail->pkt.type == MPIDI_CH3_PKT_CAS || + tail->pkt.type == MPIDI_CH3_PKT_FOP || tail->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) { /* last operation sends a response message. no need to wait * for an additional rma done pkt */ *wait_for_rma_done_pkt = 0; @@ -1952,7 +1943,7 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list); while (curr_ptr != NULL) { - if (curr_ptr->type == MPIDI_RMA_GET) { + if (curr_ptr->pkt.type == MPIDI_CH3_PKT_GET) { /* Found a GET, move it to the end */ *wait_for_rma_done_pkt = 0; @@ -1979,10 +1970,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list); - if (curr_ptr != NULL) { - target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; - } - while (curr_ptr != NULL) { MPIDI_CH3_Pkt_flags_t flags = MPIDI_CH3_PKT_FLAG_NONE; @@ -2028,12 +2015,9 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, if (*wait_for_rma_done_pkt) { flags |= MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK; } - - source_win_handle = win_ptr->handle; } - mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, source_win_handle, - target_win_handle); + mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -2185,6 +2169,9 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_put_unlock_t *lock_put_unlock_pkt = &upkt.lock_put_unlock; MPIDI_CH3_Pkt_lock_accum_unlock_t *lock_accum_unlock_pkt = &upkt.lock_accum_unlock; + MPIDI_CH3_Pkt_put_t *put_pkt; + MPIDI_CH3_Pkt_accum_t *accum_pkt; + MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt; MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_PUT_OR_ACC); @@ -2194,7 +2181,9 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) rma_op = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list); - if (rma_op->type == MPIDI_RMA_PUT) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT) { + put_pkt = &rma_op->pkt.put; + MPIDI_Pkt_init(lock_put_unlock_pkt, MPIDI_CH3_PKT_LOCK_PUT_UNLOCK); lock_put_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK | MPIDI_CH3_PKT_FLAG_RMA_UNLOCK | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK; @@ -2203,18 +2192,17 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) lock_put_unlock_pkt->lock_type = lock_type; lock_put_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; - lock_put_unlock_pkt->addr = - (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - - lock_put_unlock_pkt->count = rma_op->target_count; - lock_put_unlock_pkt->datatype = rma_op->target_datatype; + lock_put_unlock_pkt->addr = put_pkt->addr; + lock_put_unlock_pkt->count = put_pkt->count; + lock_put_unlock_pkt->datatype = put_pkt->datatype; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_put_unlock_pkt; iov[0].MPID_IOV_LEN = sizeof(*lock_put_unlock_pkt); } - else if (rma_op->type == MPIDI_RMA_ACCUMULATE) { + else if (rma_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + accum_pkt = &rma_op->pkt.accum; + MPIDI_Pkt_init(lock_accum_unlock_pkt, MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK); lock_accum_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK | MPIDI_CH3_PKT_FLAG_RMA_UNLOCK | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK; @@ -2223,18 +2211,17 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) lock_accum_unlock_pkt->lock_type = lock_type; lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; - lock_accum_unlock_pkt->addr = - (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - - lock_accum_unlock_pkt->count = rma_op->target_count; - lock_accum_unlock_pkt->datatype = rma_op->target_datatype; - lock_accum_unlock_pkt->op = rma_op->op; + lock_accum_unlock_pkt->addr = accum_pkt->addr; + lock_accum_unlock_pkt->count = accum_pkt->count; + lock_accum_unlock_pkt->datatype = accum_pkt->datatype; + lock_accum_unlock_pkt->op = accum_pkt->op; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_accum_unlock_pkt; iov[0].MPID_IOV_LEN = sizeof(*lock_accum_unlock_pkt); } - else if (rma_op->type == MPIDI_RMA_ACC_CONTIG) { + else if (rma_op->pkt.type == MPIDI_CH3_PKT_ACCUM_IMMED) { + accumi_pkt = &rma_op->pkt.accum_immed; + MPIDI_Pkt_init(lock_accum_unlock_pkt, MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK); lock_accum_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK | MPIDI_CH3_PKT_FLAG_RMA_UNLOCK | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK; @@ -2243,13 +2230,10 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) lock_accum_unlock_pkt->lock_type = lock_type; lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; - lock_accum_unlock_pkt->addr = - (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - - lock_accum_unlock_pkt->count = rma_op->target_count; - lock_accum_unlock_pkt->datatype = rma_op->target_datatype; - lock_accum_unlock_pkt->op = rma_op->op; + lock_accum_unlock_pkt->addr = accumi_pkt->addr; + lock_accum_unlock_pkt->count = accumi_pkt->count; + lock_accum_unlock_pkt->datatype = accumi_pkt->datatype; + lock_accum_unlock_pkt->op = accumi_pkt->op; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_accum_unlock_pkt; iov[0].MPID_IOV_LEN = sizeof(*lock_accum_unlock_pkt); @@ -2375,6 +2359,7 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) MPID_Datatype *dtp; MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_get_unlock_t *lock_get_unlock_pkt = &upkt.lock_get_unlock; + MPIDI_CH3_Pkt_get_t *get_pkt; MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_GET); @@ -2408,6 +2393,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) * request is freed. */ } + get_pkt = &rma_op->pkt.get; + MPIDI_Pkt_init(lock_get_unlock_pkt, MPIDI_CH3_PKT_LOCK_GET_UNLOCK); lock_get_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK | MPIDI_CH3_PKT_FLAG_RMA_UNLOCK; /* FIXME | MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK; */ lock_get_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank]; @@ -2415,12 +2402,9 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) lock_get_unlock_pkt->lock_type = lock_type; lock_get_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; - lock_get_unlock_pkt->addr = - (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - - lock_get_unlock_pkt->count = rma_op->target_count; - lock_get_unlock_pkt->datatype = rma_op->target_datatype; + lock_get_unlock_pkt->addr = get_pkt->addr; + lock_get_unlock_pkt->count = get_pkt->count; + lock_get_unlock_pkt->datatype = get_pkt->datatype; lock_get_unlock_pkt->request_handle = rreq->handle; comm_ptr = win_ptr->comm_ptr; http://git.mpich.org/mpich.git/commitdiff/ba1a400c4de2df8a662f0f9f856d824ed… commit ba1a400c4de2df8a662f0f9f856d824ed05ec221 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Nov 2 21:50:07 2014 -0600 Rename ACK packets in RMA. The packet type MPIDI_CH3_PKT_PT_RMA_DONE is used for ACK of FLUSH / UNLOCK packets. Here we rename it to MPIDI_CH3_PKT_FLUSH_ACK and modify the related functions and data structures. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 0374fe8..8f18aa3 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1809,7 +1809,7 @@ int MPIDI_CH3_PktHandler_Unlock( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, MPIDI_msg_sz_t *, MPID_Request ** ); int MPIDI_CH3_PktHandler_Flush( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, MPIDI_msg_sz_t *, MPID_Request ** ); -int MPIDI_CH3_PktHandler_PtRMADone( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, +int MPIDI_CH3_PktHandler_FlushAck( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, MPIDI_msg_sz_t *, MPID_Request ** ); int MPIDI_CH3_PktHandler_LockPutUnlock( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, MPIDI_msg_sz_t *, MPID_Request ** ); diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 6d3bed1..da2b500 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -88,7 +88,7 @@ typedef enum { MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, - MPIDI_CH3_PKT_PT_RMA_DONE, + MPIDI_CH3_PKT_FLUSH_ACK, MPIDI_CH3_PKT_LOCK_PUT_UNLOCK, /* optimization for single puts */ MPIDI_CH3_PKT_LOCK_GET_UNLOCK, /* optimization for single gets */ MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK, /* optimization for single accumulates */ @@ -326,11 +326,11 @@ typedef struct MPIDI_CH3_Pkt_lock { typedef struct MPIDI_CH3_Pkt_lock_granted { MPIDI_CH3_Pkt_type_t type; MPI_Win source_win_handle; - int target_rank; /* Used in pt_rma_done response to look up the + int target_rank; /* Used in flush_ack response to look up the * target state at the origin. */ } MPIDI_CH3_Pkt_lock_granted_t; -typedef MPIDI_CH3_Pkt_lock_granted_t MPIDI_CH3_Pkt_pt_rma_done_t; +typedef MPIDI_CH3_Pkt_lock_granted_t MPIDI_CH3_Pkt_flush_ack_t; typedef MPIDI_CH3_Pkt_lock_t MPIDI_CH3_Pkt_unlock_t; typedef MPIDI_CH3_Pkt_lock_t MPIDI_CH3_Pkt_flush_t; @@ -406,7 +406,7 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; - MPIDI_CH3_Pkt_pt_rma_done_t pt_rma_done; + MPIDI_CH3_Pkt_flush_ack_t flush_ack; MPIDI_CH3_Pkt_lock_put_unlock_t lock_put_unlock; MPIDI_CH3_Pkt_lock_get_unlock_t lock_get_unlock; MPIDI_CH3_Pkt_lock_accum_unlock_t lock_accum_unlock; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index aedbaf3..fd66526 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -150,27 +150,27 @@ static inline int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * w #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt +#define FUNCNAME MPIDI_CH3I_Send_flush_ack_pkt #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, +static inline int MPIDI_CH3I_Send_flush_ack_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, MPI_Win source_win_handle) { MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done; + MPIDI_CH3_Pkt_flush_ack_t *flush_ack_pkt = &upkt.flush_ack; MPID_Request *req; int mpi_errno=MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_FLUSH_ACK_PKT); - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_FLUSH_ACK_PKT); - MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE); - pt_rma_done_pkt->source_win_handle = source_win_handle; - pt_rma_done_pkt->target_rank = win_ptr->comm_ptr->rank; + MPIDI_Pkt_init(flush_ack_pkt, MPIDI_CH3_PKT_FLUSH_ACK); + flush_ack_pkt->source_win_handle = source_win_handle; + flush_ack_pkt->target_rank = win_ptr->comm_ptr->rank; /* Because this is in a packet handler, it is already within a critical section */ /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */ - mpi_errno = MPIDI_CH3_iStartMsg(vc, pt_rma_done_pkt, sizeof(*pt_rma_done_pkt), &req); + mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_ack_pkt, sizeof(*flush_ack_pkt), &req); /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */ if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg"); @@ -182,7 +182,7 @@ static inline int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ } fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_FLUSH_ACK_PKT); return mpi_errno; } diff --git a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c index f60ba73..755f5e2 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c @@ -586,8 +586,8 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[], MPIDI_CH3_PktHandler_Unlock; pktArray[MPIDI_CH3_PKT_FLUSH] = MPIDI_CH3_PktHandler_Flush; - pktArray[MPIDI_CH3_PKT_PT_RMA_DONE] = - MPIDI_CH3_PktHandler_PtRMADone; + pktArray[MPIDI_CH3_PKT_FLUSH_ACK] = + MPIDI_CH3_PktHandler_FlushAck; pktArray[MPIDI_CH3_PKT_LOCK_PUT_UNLOCK] = MPIDI_CH3_PktHandler_LockPutUnlock; pktArray[MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK] = diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 2ea1074..5a8de15 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -936,7 +936,7 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) MPIDI_VC_t *vc; MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc); mpi_errno = - MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, + MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, lock_queue->source_win_handle); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 1b930e9..9e88165 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -1363,37 +1363,38 @@ int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } #undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_PtRMADone +#define FUNCNAME MPIDI_CH3_PktHandler_FlushAck #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_PtRMADone(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPIDI_CH3_PktHandler_FlushAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { - MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &pkt->pt_rma_done; + MPIDI_CH3_Pkt_flush_ack_t *flush_ack_pkt = &pkt->flush_ack; MPID_Win *win_ptr = NULL; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSHACK); - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSHACK); MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt"); *buflen = sizeof(MPIDI_CH3_Pkt_t); - MPID_Win_get_ptr(pt_rma_done_pkt->source_win_handle, win_ptr); - MPIU_Assert(win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state != + MPID_Win_get_ptr(flush_ack_pkt->source_win_handle, win_ptr); + MPIU_Assert(win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_NONE); - if (win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state == + if (win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state == MPIDI_CH3_WIN_LOCK_FLUSH) - win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = + win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; else - win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; + win_ptr->targets[flush_ack_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; *rreqp = NULL; MPIDI_CH3_Progress_signal_completion(); - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSHACK); + fn_exit: return MPI_SUCCESS; } @@ -1565,7 +1566,7 @@ int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_r if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) { MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL); - mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, source_win_handle); + mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, source_win_handle); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1740,9 +1741,9 @@ int MPIDI_CH3_PktPrint_LockGetUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) return MPI_SUCCESS; } -int MPIDI_CH3_PktPrint_PtRMADone(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +int MPIDI_CH3_PktPrint_FlushAck(FILE * fp, MPIDI_CH3_Pkt_t * pkt) { - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PT_RMA_DONE\n")); + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_FLUSH_ACK\n")); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle)); return MPI_SUCCESS; } diff --git a/src/mpid/ch3/src/mpidi_printf.c b/src/mpid/ch3/src/mpidi_printf.c index 1876071..8e7d69d 100644 --- a/src/mpid/ch3/src/mpidi_printf.c +++ b/src/mpid/ch3/src/mpidi_printf.c @@ -152,8 +152,8 @@ void MPIDI_DBG_Print_packet(MPIDI_CH3_Pkt_t *pkt) case MPIDI_CH3_PKT_LOCK_GET_UNLOCK: MPIDI_CH3_PktPrint_LockGetUnlock( stdout, pkt ); break; - case MPIDI_CH3_PKT_PT_RMA_DONE: - MPIDI_CH3_PktPrint_PtRMADone( stdout, pkt ); + case MPIDI_CH3_PKT_FLUSH_ACK: + MPIDI_CH3_PktPrint_FlushAck( stdout, pkt ); break; case MPIDI_CH3_PKT_LOCK_GRANTED: MPIDI_CH3_PktPrint_LockGranted( stdout, pkt ); @@ -364,11 +364,11 @@ const char *MPIDI_Pkt_GetDescString( MPIDI_CH3_Pkt_t *pkt ) pkt->lock_get_unlock.target_win_handle, pkt->lock_get_unlock.request_handle ); break; - case MPIDI_CH3_PKT_PT_RMA_DONE: + case MPIDI_CH3_PKT_FLUSH_ACK: /* There is no rma_done packet type */ MPIU_Snprintf( pktmsg, sizeof(pktmsg), "RMA_DONE - 0x%08X", - pkt->lock_accum_unlock.source_win_handle ); + pkt->flush_ack.source_win_handle ); break; case MPIDI_CH3_PKT_LOCK_GRANTED: MPIU_Snprintf( pktmsg, sizeof(pktmsg), http://git.mpich.org/mpich.git/commitdiff/0eaf344b95ae93d0546c4a20f4c6bc058… commit 0eaf344b95ae93d0546c4a20f4c6bc0581a88e51 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Mon Jul 21 22:55:23 2014 -0500 Avoid using VC in RMA lock queue structure. We were adding an unnecessary dependency on VC structure declarations in the mpidpkt.h file. The required information in RMA lock queue is only the rank, but not actual VC. Here we replace VC with rank. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index af29902..234c3dc 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -102,7 +102,7 @@ typedef struct MPIDI_Win_lock_queue { struct MPIDI_Win_lock_queue *next; int lock_type; MPI_Win source_win_handle; - MPIDI_VC_t *vc; + int origin_rank; struct MPIDI_PT_single_op *pt_single_op; /* to store info for * lock-put-unlock optimization */ } MPIDI_Win_lock_queue; diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 46f2769..6d3bed1 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -320,6 +320,7 @@ typedef struct MPIDI_CH3_Pkt_lock { MPI_Win source_win_handle; int target_rank; /* Used in unluck/flush response to look up the * target state at the origin. */ + int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_lock_granted { @@ -338,6 +339,7 @@ typedef struct MPIDI_CH3_Pkt_lock_put_unlock { MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; + int origin_rank; int lock_type; void *addr; int count; @@ -349,6 +351,7 @@ typedef struct MPIDI_CH3_Pkt_lock_get_unlock { MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; + int origin_rank; int lock_type; void *addr; int count; @@ -361,6 +364,7 @@ typedef struct MPIDI_CH3_Pkt_lock_accum_unlock { MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; + int origin_rank; int lock_type; void *addr; int count; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index ee17ad7..aedbaf3 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -37,6 +37,7 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; lock_pkt->source_win_handle = win_ptr->handle; lock_pkt->lock_type = lock_type; + lock_pkt->origin_rank = win_ptr->comm_ptr->rank; win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED; win_ptr->targets[dest].remote_lock_mode = lock_type; diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 5c1b952..2ea1074 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -933,9 +933,10 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) they arrive within various packet/request handlers via MPIDI_CH3_Finish_rma_op_target(). That call cannot be used here, because it would enter this function recursively. */ - + MPIDI_VC_t *vc; + MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc); mpi_errno = - MPIDI_CH3I_Send_pt_rma_done_pkt(lock_queue->vc, win_ptr, + MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, lock_queue->source_win_handle); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -982,8 +983,10 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) else { /* send lock granted packet. */ + MPIDI_VC_t *vc; + MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc); mpi_errno = - MPIDI_CH3I_Send_lock_granted_pkt(lock_queue->vc, win_ptr, + MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, lock_queue->source_win_handle); /* dequeue entry from lock queue */ @@ -1077,6 +1080,7 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue) MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno=MPI_SUCCESS; MPI_Aint type_size; + MPIDI_VC_t *vc; MPIDI_STATE_DECL(MPID_STATE_DO_SIMPLE_GET); MPIDI_FUNC_ENTER(MPID_STATE_DO_SIMPLE_GET); @@ -1108,9 +1112,10 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue) MPID_Datatype_get_size_macro(lock_queue->pt_single_op->datatype, type_size); iov[1].MPID_IOV_LEN = lock_queue->pt_single_op->count * type_size; + MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc); /* Because this is in a packet handler, it is already within a critical section */ /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */ - mpi_errno = MPIDI_CH3_iSendv(lock_queue->vc, req, iov, 2); + mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2); /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */ /* --BEGIN ERROR HANDLING-- */ if (mpi_errno != MPI_SUCCESS) diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 7cb36a5..1b930e9 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -914,7 +914,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->next = NULL; new_ptr->lock_type = lock_pkt->lock_type; new_ptr->source_win_handle = lock_pkt->source_win_handle; - new_ptr->vc = vc; + new_ptr->origin_rank = lock_pkt->origin_rank; new_ptr->pt_single_op = NULL; } @@ -1001,7 +1001,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->next = NULL; new_ptr->lock_type = lock_put_unlock_pkt->lock_type; new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle; - new_ptr->vc = vc; + new_ptr->origin_rank = lock_put_unlock_pkt->origin_rank; new_ptr->pt_single_op->type = MPIDI_RMA_PUT; new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags; @@ -1149,7 +1149,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->next = NULL; new_ptr->lock_type = lock_get_unlock_pkt->lock_type; new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle; - new_ptr->vc = vc; + new_ptr->origin_rank = lock_get_unlock_pkt->origin_rank; new_ptr->pt_single_op->type = MPIDI_RMA_GET; new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags; @@ -1240,7 +1240,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, new_ptr->next = NULL; new_ptr->lock_type = lock_accum_unlock_pkt->lock_type; new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle; - new_ptr->vc = vc; + new_ptr->origin_rank = lock_accum_unlock_pkt->origin_rank; new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE; new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags; diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index aeec089..fde0a90 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -2201,6 +2201,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) lock_put_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank]; lock_put_unlock_pkt->source_win_handle = win_ptr->handle; lock_put_unlock_pkt->lock_type = lock_type; + lock_put_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; lock_put_unlock_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + @@ -2220,6 +2221,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) lock_accum_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank]; lock_accum_unlock_pkt->source_win_handle = win_ptr->handle; lock_accum_unlock_pkt->lock_type = lock_type; + lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; lock_accum_unlock_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + @@ -2239,6 +2241,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) lock_accum_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank]; lock_accum_unlock_pkt->source_win_handle = win_ptr->handle; lock_accum_unlock_pkt->lock_type = lock_type; + lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; lock_accum_unlock_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + @@ -2410,6 +2413,7 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) lock_get_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank]; lock_get_unlock_pkt->source_win_handle = win_ptr->handle; lock_get_unlock_pkt->lock_type = lock_type; + lock_get_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank; lock_get_unlock_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + http://git.mpich.org/mpich.git/commitdiff/cb04acb30ecb4e244973fa7c8c6033f92… commit cb04acb30ecb4e244973fa7c8c6033f92c71c89d Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Nov 2 21:47:22 2014 -0600 Inline static functions to remove compiler warnings. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 18c40cd..ee17ad7 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -19,7 +19,7 @@ int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, #define FUNCNAME send_lock_msg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) +static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) { int mpi_errno = MPI_SUCCESS; MPIDI_CH3_Pkt_t upkt; @@ -64,7 +64,7 @@ static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) #define FUNCNAME send_unlock_msg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_unlock_msg(int dest, MPID_Win * win_ptr) +static inline int send_unlock_msg(int dest, MPID_Win * win_ptr) { int mpi_errno = MPI_SUCCESS; MPIDI_CH3_Pkt_t upkt; @@ -111,7 +111,7 @@ static int send_unlock_msg(int dest, MPID_Win * win_ptr) #define FUNCNAME MPIDI_CH3I_Send_lock_granted_pkt #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle) +static inline int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle) { MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &upkt.lock_granted; @@ -152,7 +152,7 @@ static int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, #define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, +static inline int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, MPI_Win source_win_handle) { MPIDI_CH3_Pkt_t upkt; @@ -190,7 +190,7 @@ static int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, #define FUNCNAME acquire_local_lock #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int acquire_local_lock(MPID_Win * win_ptr, int lock_type) +static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type) { int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK); @@ -230,7 +230,7 @@ static int acquire_local_lock(MPID_Win * win_ptr, int lock_type) #define FUNCNAME do_accumulate_op #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int do_accumulate_op(MPID_Request *rreq) +static inline int do_accumulate_op(MPID_Request *rreq) { int mpi_errno = MPI_SUCCESS; MPI_Aint true_lb, true_extent; http://git.mpich.org/mpich.git/commitdiff/61f952c7a7f9ec2c59e496bf8275b29f1… commit 61f952c7a7f9ec2c59e496bf8275b29f12132768 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Nov 2 21:46:05 2014 -0600 Code refactoring to clean up the RMA code. Split RMA functionality into smaller files, and move functions to where they belong based on the file names. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h new file mode 100644 index 0000000..bc9fa21 --- /dev/null +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -0,0 +1,200 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#if !defined(MPID_RMA_OPLIST_H_INCLUDED) +#define MPID_RMA_OPLIST_H_INCLUDED + +#include "mpl_utlist.h" +#include "mpid_rma_types.h" + +/* Return nonzero if the RMA operations list is empty. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t * list) +{ + return *list == NULL; +} + + +/* Return a pointer to the first element in the list. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_head +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * list) +{ + return *list; +} + + +/* Return a pointer to the last element in the list. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list) +{ + return (*list) ? (*list)->prev : NULL; +} + + +/* Append an element to the tail of the RMA ops list + * + * @param IN list Pointer to the RMA ops list + * @param IN elem Pointer to the element to be appended + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_append +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) +{ + MPL_DL_APPEND(*list, elem); +} + + +/* Allocate a new element on the tail of the RMA operations list. + * + * @param IN list Pointer to the RMA ops list + * @param OUT new_ptr Pointer to the element that was allocated + * @return MPI error class + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Op_t ** new_elem) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_RMA_Op_t *tmp_ptr; + MPIU_CHKPMEM_DECL(1); + + /* FIXME: We should use a pool allocator here */ + MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t), + mpi_errno, "RMA operation entry"); + + tmp_ptr->next = NULL; + tmp_ptr->dataloop = NULL; + tmp_ptr->request = NULL; + + MPL_DL_APPEND(*list, tmp_ptr); + + *new_elem = tmp_ptr; + + fn_exit: + MPIU_CHKPMEM_COMMIT(); + return mpi_errno; + fn_fail: + MPIU_CHKPMEM_REAP(); + *new_elem = NULL; + goto fn_exit; +} + + +/* Unlink an element from the RMA ops list + * + * @param IN list Pointer to the RMA ops list + * @param IN elem Pointer to the element to be unlinked + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) +{ + MPL_DL_DELETE(*list, elem); +} + + +/* Free an element in the RMA operations list. + * + * @param IN list Pointer to the RMA ops list + * @param IN curr_ptr Pointer to the element to be freed. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Op_t * curr_ptr) +{ + MPIDI_RMA_Op_t *tmp_ptr = curr_ptr; + + MPIU_Assert(curr_ptr != NULL); + + MPL_DL_DELETE(*list, curr_ptr); + + /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */ + if (tmp_ptr->dataloop != NULL) + MPIU_Free(tmp_ptr->dataloop); + MPIU_Free(tmp_ptr); +} + + +/* Free an element in the RMA operations list. + * + * @param IN list Pointer to the RMA ops list + * @param INOUT curr_ptr Pointer to the element to be freed. Will be updated + * to point to the element following the element that + * was freed. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list, + MPIDI_RMA_Op_t ** curr_ptr) +{ + MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next; + + MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr); + *curr_ptr = next_ptr; +} + + +/* Free the entire RMA operations list. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Ops_free +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list) +{ + MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr; + + MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) { + MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr); + } +} + + +/* Retrieve the RMA ops list pointer from the window. This routine detects + * whether we are in an active or passive target epoch and returns the correct + * ops list; we use a shared list for active target and separate per-target + * lists for passive target. + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_ptr, int target) +{ + if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE || + win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) { + return &win_ptr->at_rma_ops_list; + } + else { + return &win_ptr->targets[target].rma_ops_list; + } +} + +#endif /* MPID_RMA_OPLIST_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpid_rma_shm.h similarity index 72% copy from src/mpid/ch3/include/mpidrma.h copy to src/mpid/ch3/include/mpid_rma_shm.h index 3710086..7be3520 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpid_rma_shm.h @@ -3,310 +3,12 @@ * (C) 2001 by Argonne National Laboratory. * See COPYRIGHT in top-level directory. */ -#if !defined(MPICH_MPIDRMA_H_INCLUDED) -#define MPICH_MPIDRMA_H_INCLUDED -#include "mpl_utlist.h" -#include "mpidi_ch3_impl.h" - -typedef enum MPIDI_RMA_Op_type { - MPIDI_RMA_PUT = 23, - MPIDI_RMA_GET = 24, - MPIDI_RMA_ACCUMULATE = 25, - /* REMOVED: MPIDI_RMA_LOCK = 26, */ - MPIDI_RMA_ACC_CONTIG = 27, - MPIDI_RMA_GET_ACCUMULATE = 28, - MPIDI_RMA_COMPARE_AND_SWAP = 29, - MPIDI_RMA_FETCH_AND_OP = 30 -} MPIDI_RMA_Op_type_t; - -/* Special case RMA operations */ - -enum MPIDI_RMA_Datatype { - MPIDI_RMA_DATATYPE_BASIC = 50, - MPIDI_RMA_DATATYPE_DERIVED = 51 -}; - -/* We start with an arbitrarily chosen number (42), to help with - * debugging when a packet type is not initialized or wrongly - * initialized. */ -enum MPID_Lock_state { - MPID_LOCK_NONE = 42, - MPID_LOCK_SHARED_ALL -}; - -/* - * RMA Declarations. We should move these into something separate from - * a Request. - */ - -/* to send derived datatype across in RMA ops */ -typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */ - int is_contig; - int max_contig_blocks; - MPI_Aint size; - MPI_Aint extent; - int dataloop_size; /* not needed because this info is sent in - * packet header. remove it after lock/unlock - * is implemented in the device */ - void *dataloop; /* pointer needed to update pointers - * within dataloop on remote side */ - int dataloop_depth; - int eltype; - MPI_Aint ub, lb, true_ub, true_lb; - int has_sticky_ub, has_sticky_lb; -} MPIDI_RMA_dtype_info; - -/* for keeping track of RMA ops, which will be executed at the next sync call */ -typedef struct MPIDI_RMA_Op { - struct MPIDI_RMA_Op *prev; /* pointer to next element in list */ - struct MPIDI_RMA_Op *next; /* pointer to next element in list */ - /* FIXME: It would be better to setup the packet that will be sent, at - * least in most cases (if, as a result of the sync/ops/sync sequence, - * a different packet type is needed, it can be extracted from the - * information otherwise stored). */ - MPIDI_RMA_Op_type_t type; - void *origin_addr; - int origin_count; - MPI_Datatype origin_datatype; - int target_rank; - MPI_Aint target_disp; - int target_count; - MPI_Datatype target_datatype; - MPI_Op op; /* for accumulate */ - /* Used to complete operations */ - struct MPID_Request *request; - MPIDI_RMA_dtype_info dtype_info; - void *dataloop; - void *result_addr; - int result_count; - MPI_Datatype result_datatype; - void *compare_addr; - int compare_count; - MPI_Datatype compare_datatype; -} MPIDI_RMA_Op_t; - -typedef struct MPIDI_PT_single_op { - int type; /* put, get, or accum. */ - void *addr; - int count; - MPI_Datatype datatype; - MPI_Op op; - void *data; /* for queued puts and accumulates, data is copied here */ - MPI_Request request_handle; /* for gets */ - int data_recd; /* to indicate if the data has been received */ - MPIDI_CH3_Pkt_flags_t flags; -} MPIDI_PT_single_op; - -typedef struct MPIDI_Win_lock_queue { - struct MPIDI_Win_lock_queue *next; - int lock_type; - MPI_Win source_win_handle; - MPIDI_VC_t *vc; - struct MPIDI_PT_single_op *pt_single_op; /* to store info for - * lock-put-unlock optimization */ -} MPIDI_Win_lock_queue; - -/* Routine use to tune RMA optimizations */ -void MPIDI_CH3_RMA_SetAccImmed(int flag); - -/*** RMA OPS LIST HELPER ROUTINES ***/ - -typedef MPIDI_RMA_Op_t *MPIDI_RMA_Ops_list_t; - -/* Return nonzero if the RMA operations list is empty. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t * list) -{ - return *list == NULL; -} - - -/* Return a pointer to the first element in the list. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_head -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * list) -{ - return *list; -} - - -/* Return a pointer to the last element in the list. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list) -{ - return (*list) ? (*list)->prev : NULL; -} - - -/* Append an element to the tail of the RMA ops list - * - * @param IN list Pointer to the RMA ops list - * @param IN elem Pointer to the element to be appended - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_append -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) -{ - MPL_DL_APPEND(*list, elem); -} - - -/* Allocate a new element on the tail of the RMA operations list. - * - * @param IN list Pointer to the RMA ops list - * @param OUT new_ptr Pointer to the element that was allocated - * @return MPI error class - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list, - MPIDI_RMA_Op_t ** new_elem) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_RMA_Op_t *tmp_ptr; - MPIU_CHKPMEM_DECL(1); - - /* FIXME: We should use a pool allocator here */ - MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t), - mpi_errno, "RMA operation entry"); - - tmp_ptr->next = NULL; - tmp_ptr->dataloop = NULL; - tmp_ptr->request = NULL; - - MPL_DL_APPEND(*list, tmp_ptr); - - *new_elem = tmp_ptr; - - fn_exit: - MPIU_CHKPMEM_COMMIT(); - return mpi_errno; - fn_fail: - MPIU_CHKPMEM_REAP(); - *new_elem = NULL; - goto fn_exit; -} - - -/* Unlink an element from the RMA ops list - * - * @param IN list Pointer to the RMA ops list - * @param IN elem Pointer to the element to be unlinked - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) -{ - MPL_DL_DELETE(*list, elem); -} - - -/* Free an element in the RMA operations list. - * - * @param IN list Pointer to the RMA ops list - * @param IN curr_ptr Pointer to the element to be freed. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list, - MPIDI_RMA_Op_t * curr_ptr) -{ - MPIDI_RMA_Op_t *tmp_ptr = curr_ptr; - - MPIU_Assert(curr_ptr != NULL); +#if !defined(MPID_RMA_SHM_H_INCLUDED) +#define MPID_RMA_SHM_H_INCLUDED - MPL_DL_DELETE(*list, curr_ptr); - - /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */ - if (tmp_ptr->dataloop != NULL) - MPIU_Free(tmp_ptr->dataloop); - MPIU_Free(tmp_ptr); -} - - -/* Free an element in the RMA operations list. - * - * @param IN list Pointer to the RMA ops list - * @param INOUT curr_ptr Pointer to the element to be freed. Will be updated - * to point to the element following the element that - * was freed. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list, - MPIDI_RMA_Op_t ** curr_ptr) -{ - MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next; - - MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr); - *curr_ptr = next_ptr; -} - - -/* Free the entire RMA operations list. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_free -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list) -{ - MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr; - - MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) { - MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr); - } -} - - -/* Retrieve the RMA ops list pointer from the window. This routine detects - * whether we are in an active or passive target epoch and returns the correct - * ops list; we use a shared list for active target and separate per-target - * lists for passive target. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_ptr, int target) -{ - if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE || - win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) { - return &win_ptr->at_rma_ops_list; - } - else { - return &win_ptr->targets[target].rma_ops_list; - } -} - - -/* ------------------------------------------------------------------------ */ -/* - * Followings are new routines for origin completion for RMA operations. - */ -/* ------------------------------------------------------------------------ */ +#include "mpl_utlist.h" +#include "mpid_rma_types.h" #define ASSIGN_COPY(src, dest, count, type) \ { \ @@ -1060,7 +762,4 @@ static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_ad } -#undef FUNCNAME -#undef FCNAME - -#endif +#endif /* MPID_RMA_SHM_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h new file mode 100644 index 0000000..af29902 --- /dev/null +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -0,0 +1,112 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#if !defined(MPID_RMA_TYPES_H_INCLUDED) +#define MPID_RMA_TYPES_H_INCLUDED + +#include "mpidi_ch3_impl.h" + +typedef enum MPIDI_RMA_Op_type { + MPIDI_RMA_PUT = 23, + MPIDI_RMA_GET = 24, + MPIDI_RMA_ACCUMULATE = 25, + /* REMOVED: MPIDI_RMA_LOCK = 26, */ + MPIDI_RMA_ACC_CONTIG = 27, + MPIDI_RMA_GET_ACCUMULATE = 28, + MPIDI_RMA_COMPARE_AND_SWAP = 29, + MPIDI_RMA_FETCH_AND_OP = 30 +} MPIDI_RMA_Op_type_t; + +/* Special case RMA operations */ + +enum MPIDI_RMA_Datatype { + MPIDI_RMA_DATATYPE_BASIC = 50, + MPIDI_RMA_DATATYPE_DERIVED = 51 +}; + +/* We start with an arbitrarily chosen number (42), to help with + * debugging when a packet type is not initialized or wrongly + * initialized. */ +enum MPID_Lock_state { + MPID_LOCK_NONE = 42, + MPID_LOCK_SHARED_ALL +}; + +/* + * RMA Declarations. We should move these into something separate from + * a Request. + */ + +/* to send derived datatype across in RMA ops */ +typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */ + int is_contig; + int max_contig_blocks; + MPI_Aint size; + MPI_Aint extent; + int dataloop_size; /* not needed because this info is sent in + * packet header. remove it after lock/unlock + * is implemented in the device */ + void *dataloop; /* pointer needed to update pointers + * within dataloop on remote side */ + int dataloop_depth; + int eltype; + MPI_Aint ub, lb, true_ub, true_lb; + int has_sticky_ub, has_sticky_lb; +} MPIDI_RMA_dtype_info; + +/* for keeping track of RMA ops, which will be executed at the next sync call */ +typedef struct MPIDI_RMA_Op { + struct MPIDI_RMA_Op *prev; /* pointer to next element in list */ + struct MPIDI_RMA_Op *next; /* pointer to next element in list */ + /* FIXME: It would be better to setup the packet that will be sent, at + * least in most cases (if, as a result of the sync/ops/sync sequence, + * a different packet type is needed, it can be extracted from the + * information otherwise stored). */ + MPIDI_RMA_Op_type_t type; + void *origin_addr; + int origin_count; + MPI_Datatype origin_datatype; + int target_rank; + MPI_Aint target_disp; + int target_count; + MPI_Datatype target_datatype; + MPI_Op op; /* for accumulate */ + /* Used to complete operations */ + struct MPID_Request *request; + MPIDI_RMA_dtype_info dtype_info; + void *dataloop; + void *result_addr; + int result_count; + MPI_Datatype result_datatype; + void *compare_addr; + int compare_count; + MPI_Datatype compare_datatype; +} MPIDI_RMA_Op_t; + +typedef struct MPIDI_PT_single_op { + int type; /* put, get, or accum. */ + void *addr; + int count; + MPI_Datatype datatype; + MPI_Op op; + void *data; /* for queued puts and accumulates, data is copied here */ + MPI_Request request_handle; /* for gets */ + int data_recd; /* to indicate if the data has been received */ + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_PT_single_op; + +typedef struct MPIDI_Win_lock_queue { + struct MPIDI_Win_lock_queue *next; + int lock_type; + MPI_Win source_win_handle; + MPIDI_VC_t *vc; + struct MPIDI_PT_single_op *pt_single_op; /* to store info for + * lock-put-unlock optimization */ +} MPIDI_Win_lock_queue; + +typedef MPIDI_RMA_Op_t *MPIDI_RMA_Ops_list_t; + +#endif /* MPID_RMA_TYPES_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 6a9db2d..0374fe8 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1228,8 +1228,6 @@ void MPIDI_CH3_RMA_Init_Pvars(void); /* internal */ int MPIDI_CH3I_Release_lock(MPID_Win * win_ptr); int MPIDI_CH3I_Try_acquire_win_lock(MPID_Win * win_ptr, int requested_lock); -int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win *win_ptr, int source_win_hdl); -int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t * vc, MPID_Win *win_ptr, int source_win_hdl); int MPIDI_CH3_Start_rma_op_target(MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags); int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t *vc, MPID_Win *win_ptr, int is_rma_update, MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle); diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 3710086..18c40cd 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -3,537 +3,102 @@ * (C) 2001 by Argonne National Laboratory. * See COPYRIGHT in top-level directory. */ -#if !defined(MPICH_MPIDRMA_H_INCLUDED) -#define MPICH_MPIDRMA_H_INCLUDED -#include "mpl_utlist.h" -#include "mpidi_ch3_impl.h" +#if !defined(MPID_RMA_H_INCLUDED) +#define MPID_RMA_H_INCLUDED -typedef enum MPIDI_RMA_Op_type { - MPIDI_RMA_PUT = 23, - MPIDI_RMA_GET = 24, - MPIDI_RMA_ACCUMULATE = 25, - /* REMOVED: MPIDI_RMA_LOCK = 26, */ - MPIDI_RMA_ACC_CONTIG = 27, - MPIDI_RMA_GET_ACCUMULATE = 28, - MPIDI_RMA_COMPARE_AND_SWAP = 29, - MPIDI_RMA_FETCH_AND_OP = 30 -} MPIDI_RMA_Op_type_t; +#include "mpid_rma_types.h" +#include "mpid_rma_oplist.h" +#include "mpid_rma_shm.h" -/* Special case RMA operations */ +int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, + MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle, + MPI_Win target_win_handle); -enum MPIDI_RMA_Datatype { - MPIDI_RMA_DATATYPE_BASIC = 50, - MPIDI_RMA_DATATYPE_DERIVED = 51 -}; - -/* We start with an arbitrarily chosen number (42), to help with - * debugging when a packet type is not initialized or wrongly - * initialized. */ -enum MPID_Lock_state { - MPID_LOCK_NONE = 42, - MPID_LOCK_SHARED_ALL -}; - -/* - * RMA Declarations. We should move these into something separate from - * a Request. - */ - -/* to send derived datatype across in RMA ops */ -typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */ - int is_contig; - int max_contig_blocks; - MPI_Aint size; - MPI_Aint extent; - int dataloop_size; /* not needed because this info is sent in - * packet header. remove it after lock/unlock - * is implemented in the device */ - void *dataloop; /* pointer needed to update pointers - * within dataloop on remote side */ - int dataloop_depth; - int eltype; - MPI_Aint ub, lb, true_ub, true_lb; - int has_sticky_ub, has_sticky_lb; -} MPIDI_RMA_dtype_info; - -/* for keeping track of RMA ops, which will be executed at the next sync call */ -typedef struct MPIDI_RMA_Op { - struct MPIDI_RMA_Op *prev; /* pointer to next element in list */ - struct MPIDI_RMA_Op *next; /* pointer to next element in list */ - /* FIXME: It would be better to setup the packet that will be sent, at - * least in most cases (if, as a result of the sync/ops/sync sequence, - * a different packet type is needed, it can be extracted from the - * information otherwise stored). */ - MPIDI_RMA_Op_type_t type; - void *origin_addr; - int origin_count; - MPI_Datatype origin_datatype; - int target_rank; - MPI_Aint target_disp; - int target_count; - MPI_Datatype target_datatype; - MPI_Op op; /* for accumulate */ - /* Used to complete operations */ - struct MPID_Request *request; - MPIDI_RMA_dtype_info dtype_info; - void *dataloop; - void *result_addr; - int result_count; - MPI_Datatype result_datatype; - void *compare_addr; - int compare_count; - MPI_Datatype compare_datatype; -} MPIDI_RMA_Op_t; - -typedef struct MPIDI_PT_single_op { - int type; /* put, get, or accum. */ - void *addr; - int count; - MPI_Datatype datatype; - MPI_Op op; - void *data; /* for queued puts and accumulates, data is copied here */ - MPI_Request request_handle; /* for gets */ - int data_recd; /* to indicate if the data has been received */ - MPIDI_CH3_Pkt_flags_t flags; -} MPIDI_PT_single_op; - -typedef struct MPIDI_Win_lock_queue { - struct MPIDI_Win_lock_queue *next; - int lock_type; - MPI_Win source_win_handle; - MPIDI_VC_t *vc; - struct MPIDI_PT_single_op *pt_single_op; /* to store info for - * lock-put-unlock optimization */ -} MPIDI_Win_lock_queue; - -/* Routine use to tune RMA optimizations */ -void MPIDI_CH3_RMA_SetAccImmed(int flag); - -/*** RMA OPS LIST HELPER ROUTINES ***/ - -typedef MPIDI_RMA_Op_t *MPIDI_RMA_Ops_list_t; - -/* Return nonzero if the RMA operations list is empty. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t * list) -{ - return *list == NULL; -} - - -/* Return a pointer to the first element in the list. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_head -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t * list) -{ - return *list; -} - - -/* Return a pointer to the last element in the list. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Op_t *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t * list) -{ - return (*list) ? (*list)->prev : NULL; -} - - -/* Append an element to the tail of the RMA ops list - * - * @param IN list Pointer to the RMA ops list - * @param IN elem Pointer to the element to be appended - */ #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_append +#define FUNCNAME send_lock_msg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) +static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) { - MPL_DL_APPEND(*list, elem); -} + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_lock_t *lock_pkt = &upkt.lock; + MPID_Request *req = NULL; + MPIDI_VC_t *vc; + MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_MSG); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_LOCK_MSG); + MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED); -/* Allocate a new element on the tail of the RMA operations list. - * - * @param IN list Pointer to the RMA ops list - * @param OUT new_ptr Pointer to the element that was allocated - * @return MPI error class - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t * list, - MPIDI_RMA_Op_t ** new_elem) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_RMA_Op_t *tmp_ptr; - MPIU_CHKPMEM_DECL(1); + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); - /* FIXME: We should use a pool allocator here */ - MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_Op_t *, sizeof(MPIDI_RMA_Op_t), - mpi_errno, "RMA operation entry"); + MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); + lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; + lock_pkt->source_win_handle = win_ptr->handle; + lock_pkt->lock_type = lock_type; - tmp_ptr->next = NULL; - tmp_ptr->dataloop = NULL; - tmp_ptr->request = NULL; + win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED; + win_ptr->targets[dest].remote_lock_mode = lock_type; - MPL_DL_APPEND(*list, tmp_ptr); + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_pkt, sizeof(*lock_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - *new_elem = tmp_ptr; + /* release the request returned by iStartMsg */ + if (req != NULL) { + MPID_Request_release(req); + } fn_exit: - MPIU_CHKPMEM_COMMIT(); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_MSG); return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ fn_fail: - MPIU_CHKPMEM_REAP(); - *new_elem = NULL; goto fn_exit; + /* --END ERROR HANDLING-- */ } - -/* Unlink an element from the RMA ops list - * - * @param IN list Pointer to the RMA ops list - * @param IN elem Pointer to the element to be unlinked - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list, MPIDI_RMA_Op_t * elem) -{ - MPL_DL_DELETE(*list, elem); -} - - -/* Free an element in the RMA operations list. - * - * @param IN list Pointer to the RMA ops list - * @param IN curr_ptr Pointer to the element to be freed. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t * list, - MPIDI_RMA_Op_t * curr_ptr) -{ - MPIDI_RMA_Op_t *tmp_ptr = curr_ptr; - - MPIU_Assert(curr_ptr != NULL); - - MPL_DL_DELETE(*list, curr_ptr); - - /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */ - if (tmp_ptr->dataloop != NULL) - MPIU_Free(tmp_ptr->dataloop); - MPIU_Free(tmp_ptr); -} - - -/* Free an element in the RMA operations list. - * - * @param IN list Pointer to the RMA ops list - * @param INOUT curr_ptr Pointer to the element to be freed. Will be updated - * to point to the element following the element that - * was freed. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t * list, - MPIDI_RMA_Op_t ** curr_ptr) -{ - MPIDI_RMA_Op_t *next_ptr = (*curr_ptr)->next; - - MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr); - *curr_ptr = next_ptr; -} - - -/* Free the entire RMA operations list. - */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Ops_free -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t * list) -{ - MPIDI_RMA_Op_t *curr_ptr, *tmp_ptr; - - MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) { - MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr); - } -} - - -/* Retrieve the RMA ops list pointer from the window. This routine detects - * whether we are in an active or passive target epoch and returns the correct - * ops list; we use a shared list for active target and separate per-target - * lists for passive target. - */ #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Get_ops_list +#define FUNCNAME send_unlock_msg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline MPIDI_RMA_Ops_list_t *MPIDI_CH3I_RMA_Get_ops_list(MPID_Win * win_ptr, int target) -{ - if (win_ptr->epoch_state == MPIDI_EPOCH_FENCE || - win_ptr->epoch_state == MPIDI_EPOCH_START || win_ptr->epoch_state == MPIDI_EPOCH_PSCW) { - return &win_ptr->at_rma_ops_list; - } - else { - return &win_ptr->targets[target].rma_ops_list; - } -} - - -/* ------------------------------------------------------------------------ */ -/* - * Followings are new routines for origin completion for RMA operations. - */ -/* ------------------------------------------------------------------------ */ - -#define ASSIGN_COPY(src, dest, count, type) \ - { \ - type *src_ = (type *) src; \ - type *dest_ = (type *) dest; \ - int i; \ - for (i = 0; i < count; i++) \ - dest_[i] = src_[i]; \ - goto fn_exit; \ - } - -static inline int shm_copy(const void *src, int scount, MPI_Datatype stype, - void *dest, int dcount, MPI_Datatype dtype) +static int send_unlock_msg(int dest, MPID_Win * win_ptr) { int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &upkt.unlock; + MPID_Request *req = NULL; + MPIDI_VC_t *vc; + MPIDI_STATE_DECL(MPID_STATE_SEND_UNLOCK_MSG); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_UNLOCK_MSG); - /* We use a threshold of operations under which a for loop of assignments is - * used. Even though this happens at smaller block lengths, making it - * potentially inefficient, it can take advantage of some vectorization - * available on most modern processors. */ -#define SHM_OPS_THRESHOLD (16) - - if (MPIR_DATATYPE_IS_PREDEFINED(stype) && MPIR_DATATYPE_IS_PREDEFINED(dtype) && - scount <= SHM_OPS_THRESHOLD) { - - /* FIXME: We currently only optimize a few predefined datatypes, which - * have a direct C datatype mapping. */ - - /* The below list of datatypes is based on those specified in the MPI-3 - * standard on page 665. */ - switch (stype) { - case MPI_CHAR: - ASSIGN_COPY(src, dest, scount, char); - - case MPI_SHORT: - ASSIGN_COPY(src, dest, scount, signed short int); - - case MPI_INT: - ASSIGN_COPY(src, dest, scount, signed int); - - case MPI_LONG: - ASSIGN_COPY(src, dest, scount, signed long int); - - case MPI_LONG_LONG_INT: /* covers MPI_LONG_LONG too */ - ASSIGN_COPY(src, dest, scount, signed long long int); - - case MPI_SIGNED_CHAR: - ASSIGN_COPY(src, dest, scount, signed char); - - case MPI_UNSIGNED_CHAR: - ASSIGN_COPY(src, dest, scount, unsigned char); - - case MPI_UNSIGNED_SHORT: - ASSIGN_COPY(src, dest, scount, unsigned short int); - - case MPI_UNSIGNED: - ASSIGN_COPY(src, dest, scount, unsigned int); - - case MPI_UNSIGNED_LONG: - ASSIGN_COPY(src, dest, scount, unsigned long int); - - case MPI_UNSIGNED_LONG_LONG: - ASSIGN_COPY(src, dest, scount, unsigned long long int); - - case MPI_FLOAT: - ASSIGN_COPY(src, dest, scount, float); - - case MPI_DOUBLE: - ASSIGN_COPY(src, dest, scount, double); - - case MPI_LONG_DOUBLE: - ASSIGN_COPY(src, dest, scount, long double); - -#if 0 - /* FIXME: we need a configure check to define HAVE_WCHAR_T before - * this can be enabled */ - case MPI_WCHAR: - ASSIGN_COPY(src, dest, scount, wchar_t); -#endif - -#if 0 - /* FIXME: we need a configure check to define HAVE_C_BOOL before - * this can be enabled */ - case MPI_C_BOOL: - ASSIGN_COPY(src, dest, scount, _Bool); -#endif - -#if HAVE_INT8_T - case MPI_INT8_T: - ASSIGN_COPY(src, dest, scount, int8_t); -#endif /* HAVE_INT8_T */ - -#if HAVE_INT16_T - case MPI_INT16_T: - ASSIGN_COPY(src, dest, scount, int16_t); -#endif /* HAVE_INT16_T */ - -#if HAVE_INT32_T - case MPI_INT32_T: - ASSIGN_COPY(src, dest, scount, int32_t); -#endif /* HAVE_INT32_T */ - -#if HAVE_INT64_T - case MPI_INT64_T: - ASSIGN_COPY(src, dest, scount, int64_t); -#endif /* HAVE_INT64_T */ - -#if HAVE_UINT8_T - case MPI_UINT8_T: - ASSIGN_COPY(src, dest, scount, uint8_t); -#endif /* HAVE_UINT8_T */ - -#if HAVE_UINT16_T - case MPI_UINT16_T: - ASSIGN_COPY(src, dest, scount, uint16_t); -#endif /* HAVE_UINT16_T */ - -#if HAVE_UINT32_T - case MPI_UINT32_T: - ASSIGN_COPY(src, dest, scount, uint32_t); -#endif /* HAVE_UINT32_T */ - -#if HAVE_UINT64_T - case MPI_UINT64_T: - ASSIGN_COPY(src, dest, scount, uint64_t); -#endif /* HAVE_UINT64_T */ - - case MPI_AINT: - ASSIGN_COPY(src, dest, scount, MPI_Aint); - - case MPI_COUNT: - ASSIGN_COPY(src, dest, scount, MPI_Count); - - case MPI_OFFSET: - ASSIGN_COPY(src, dest, scount, MPI_Offset); - -#if 0 - /* FIXME: we need a configure check to define HAVE_C_COMPLEX before - * this can be enabled */ - case MPI_C_COMPLEX: /* covers MPI_C_FLOAT_COMPLEX as well */ - ASSIGN_COPY(src, dest, scount, float _Complex); -#endif - -#if 0 - /* FIXME: we need a configure check to define HAVE_C_DOUPLE_COMPLEX - * before this can be enabled */ - case MPI_C_DOUBLE_COMPLEX: - ASSIGN_COPY(src, dest, scount, double _Complex); -#endif - -#if 0 - /* FIXME: we need a configure check to define - * HAVE_C_LONG_DOUPLE_COMPLEX before this can be enabled */ - case MPI_C_LONG_DOUBLE_COMPLEX: - ASSIGN_COPY(src, dest, scount, long double _Complex); -#endif - -#if 0 - /* Types that don't have a direct equivalent */ - case MPI_BYTE: - case MPI_PACKED: -#endif - -#if 0 /* Fortran types */ - case MPI_INTEGER: - case MPI_REAL: - case MPI_DOUBLE_PRECISION: - case MPI_COMPLEX: - case MPI_LOGICAL: - case MPI_CHARACTER: -#endif - -#if 0 /* C++ types */ - case MPI_CXX_BOOL: - case MPI_CXX_FLOAT_COMPLEX: - case MPI_CXX_DOUBLE_COMPLEX: - case MPI_CXX_LONG_DOUBLE_COMPLEX: -#endif + MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_GRANTED); -#if 0 /* Optional Fortran types */ - case MPI_DOUBLE_COMPLEX: - case MPI_INTEGER1: - case MPI_INTEGER2: - case MPI_INTEGER4: - case MPI_INTEGER8: - case MPI_INTEGER16: - case MPI_REAL2: - case MPI_REAL4: - case MPI_REAL8: - case MPI_REAL16: - case MPI_COMPLEX4: - case MPI_COMPLEX8: - case MPI_COMPLEX16: - case MPI_COMPLEX32: -#endif + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); -#if 0 /* C datatypes for reduction functions */ - case MPI_FLOAT_INT: - case MPI_DOUBLE_INT: - case MPI_LONG_INT: - case MPI_2INT: - case MPI_LONG_DOUBLE_INT: -#endif + /* Send a lock packet over to the target. wait for the lock_granted + * reply. Then do all the RMA ops. */ -#if 0 /* Fortran datatypes for reduction functions */ - case MPI_2REAL: - case MPI_2DOUBLE_PRECISION: - case MPI_2INTEGER: -#endif + MPIDI_Pkt_init(unlock_pkt, MPIDI_CH3_PKT_UNLOCK); + unlock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; -#if 0 /* Random types not present in the standard */ - case MPI_2COMPLEX: - case MPI_2DOUBLE_COMPLEX: -#endif + /* Reset the local state of the target to unlocked */ + win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; - default: - /* Just to make sure the switch statement is not empty */ - ; - } - } + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, unlock_pkt, sizeof(*unlock_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - mpi_errno = MPIR_Localcopy(src, scount, stype, dest, dcount, dtype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); + /* Release the request returned by iStartMsg */ + if (req != NULL) { + MPID_Request_release(req); } fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_UNLOCK_MSG); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: @@ -541,526 +106,274 @@ static inline int shm_copy(const void *src, int scount, MPI_Datatype stype, /* --END ERROR HANDLING-- */ } + #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Shm_put_op +#define FUNCNAME MPIDI_CH3I_Send_lock_granted_pkt #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Shm_put_op(const void *origin_addr, int origin_count, MPI_Datatype - origin_datatype, int target_rank, MPI_Aint target_disp, - int target_count, MPI_Datatype target_datatype, - MPID_Win * win_ptr) +static int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle) { - int mpi_errno = MPI_SUCCESS; - void *base = NULL; - int disp_unit; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP); - - if (win_ptr->shm_allocated == TRUE) { - base = win_ptr->shm_base_addrs[target_rank]; - disp_unit = win_ptr->disp_units[target_rank]; - } - else { - base = win_ptr->base; - disp_unit = win_ptr->disp_unit; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &upkt.lock_granted; + MPID_Request *req = NULL; + int mpi_errno; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT); + + /* send lock granted packet */ + MPIDI_Pkt_init(lock_granted_pkt, MPIDI_CH3_PKT_LOCK_GRANTED); + lock_granted_pkt->source_win_handle = source_win_handle; + lock_granted_pkt->target_rank = win_ptr->comm_ptr->rank; + + MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, + (MPIU_DBG_FDEST, "sending lock granted pkt on vc=%p, source_win_handle=%#08x", + vc, lock_granted_pkt->source_win_handle)); + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_granted_pkt, sizeof(*lock_granted_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + if (mpi_errno) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); } - mpi_errno = shm_copy(origin_addr, origin_count, origin_datatype, - (char *) base + disp_unit * target_disp, target_count, target_datatype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); + if (req != NULL) { + MPID_Request_release(req); } - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_PUT_OP); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT); + + return mpi_errno; } #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Shm_acc_op +#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_count, MPI_Datatype - origin_datatype, int target_rank, MPI_Aint target_disp, - int target_count, MPI_Datatype target_datatype, MPI_Op op, - MPID_Win * win_ptr) +static int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, + MPI_Win source_win_handle) { - void *base = NULL; - int disp_unit, shm_op = 0; - MPI_User_function *uop = NULL; - MPID_Datatype *dtp; - int mpi_errno = MPI_SUCCESS; - MPIU_CHKLMEM_DECL(2); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP); + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done; + MPID_Request *req; + int mpi_errno=MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP); + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); - if (win_ptr->shm_allocated == TRUE) { - shm_op = 1; - base = win_ptr->shm_base_addrs[target_rank]; - disp_unit = win_ptr->disp_units[target_rank]; - } - else { - base = win_ptr->base; - disp_unit = win_ptr->disp_unit; - } + MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE); + pt_rma_done_pkt->source_win_handle = source_win_handle; + pt_rma_done_pkt->target_rank = win_ptr->comm_ptr->rank; - if (op == MPI_REPLACE) { - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - mpi_errno = shm_copy(origin_addr, origin_count, origin_datatype, - (char *) base + disp_unit * target_disp, target_count, - target_datatype); - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - goto fn_exit; + /* Because this is in a packet handler, it is already within a critical section */ + /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */ + mpi_errno = MPIDI_CH3_iStartMsg(vc, pt_rma_done_pkt, sizeof(*pt_rma_done_pkt), &req); + /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */ + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg"); } - MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN), - mpi_errno, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", op); - - /* get the function by indexing into the op table */ - uop = MPIR_OP_HDL_TO_FN(op); - - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - /* Cast away const'ness for origin_address in order to - * avoid changing the prototype for MPI_User_function */ - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - (*uop) ((void *) origin_addr, (char *) base + disp_unit * target_disp, - &target_count, &target_datatype); - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (req != NULL) + { + MPID_Request_release(req); } - else { - /* derived datatype */ - MPID_Segment *segp; - DLOOP_VECTOR *dloop_vec; - MPI_Aint first, last; - int vec_len, i, type_size, count; - MPI_Datatype type; - MPI_Aint true_lb, true_extent, extent; - void *tmp_buf = NULL, *target_buf; - const void *source_buf; - - if (origin_datatype != target_datatype) { - /* first copy the data into a temporary buffer with - * the same datatype as the target. Then do the - * accumulate operation. */ - - MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent); - MPID_Datatype_get_extent_macro(target_datatype, extent); - - MPIU_CHKLMEM_MALLOC(tmp_buf, void *, - target_count * (MPIR_MAX(extent, true_extent)), - mpi_errno, "temporary buffer"); - /* adjust for potential negative lower bound in datatype */ - tmp_buf = (void *) ((char *) tmp_buf - true_lb); - - mpi_errno = MPIR_Localcopy(origin_addr, origin_count, - origin_datatype, tmp_buf, target_count, target_datatype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } - - if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - /* target predefined type, origin derived datatype */ - - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - (*uop) (tmp_buf, (char *) base + disp_unit * target_disp, - &target_count, &target_datatype); - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - } - else { - - segp = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER, - "**nomem", "**nomem %s", "MPID_Segment_alloc"); - MPID_Segment_init(NULL, target_count, target_datatype, segp, 0); - first = 0; - last = SEGMENT_IGNORE_LAST; - - MPID_Datatype_get_ptr(target_datatype, dtp); - vec_len = dtp->max_contig_blocks * target_count + 1; - /* +1 needed because Rob says so */ - MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *, - vec_len * sizeof(DLOOP_VECTOR), mpi_errno, "dloop vector"); - - MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len); + fn_fail: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); + return mpi_errno; +} - source_buf = (tmp_buf != NULL) ? (const void *) tmp_buf : origin_addr; - target_buf = (char *) base + disp_unit * target_disp; - type = dtp->eltype; - type_size = MPID_Datatype_get_basic_size(type); - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - for (i = 0; i < vec_len; i++) { - MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int); - (*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), - (char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), - &count, &type); +#undef FUNCNAME +#define FUNCNAME acquire_local_lock +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int acquire_local_lock(MPID_Win * win_ptr, int lock_type) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_ACQUIRE_LOCAL_LOCK); + + /* poke the progress engine until the local lock is granted */ + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { + MPID_Progress_state progress_state; + + MPID_Progress_start(&progress_state); + while (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { + mpi_errno = MPID_Progress_wait(&progress_state); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPID_Progress_end(&progress_state); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); } - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - - MPID_Segment_free(segp); + /* --END ERROR HANDLING-- */ } + MPID_Progress_end(&progress_state); } + win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; + win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_mode = lock_type; + fn_exit: - MPIU_CHKLMEM_FREEALL(); - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_ACC_OP); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_ACQUIRE_LOCAL_LOCK); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: - if (shm_op) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); goto fn_exit; /* --END ERROR HANDLING-- */ } #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Shm_get_acc_op +#define FUNCNAME do_accumulate_op #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_count, MPI_Datatype - origin_datatype, void *result_addr, int result_count, - MPI_Datatype result_datatype, int target_rank, MPI_Aint - target_disp, int target_count, - MPI_Datatype target_datatype, MPI_Op op, - MPID_Win * win_ptr) +static int do_accumulate_op(MPID_Request *rreq) { - int disp_unit, shm_locked = 0; - void *base = NULL; - MPI_User_function *uop = NULL; - MPID_Datatype *dtp; int mpi_errno = MPI_SUCCESS; - MPIU_CHKLMEM_DECL(2); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP); - - if (win_ptr->shm_allocated == TRUE) { - base = win_ptr->shm_base_addrs[target_rank]; - disp_unit = win_ptr->disp_units[target_rank]; - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - shm_locked = 1; - } - else { - base = win_ptr->base; - disp_unit = win_ptr->disp_unit; - } - - /* Perform the local get first, then the accumulate */ - mpi_errno = shm_copy((char *) base + disp_unit * target_disp, target_count, target_datatype, - result_addr, result_count, result_datatype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - /* NO_OP: Don't perform the accumulate */ - if (op == MPI_NO_OP) { - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - shm_locked = 0; - } - - goto fn_exit; - } - - if (op == MPI_REPLACE) { - mpi_errno = shm_copy(origin_addr, origin_count, origin_datatype, - (char *) base + disp_unit * target_disp, target_count, - target_datatype); - + MPI_Aint true_lb, true_extent; + MPI_User_function *uop; + MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP); + + MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP); + + if (rreq->dev.op == MPI_REPLACE) + { + /* simply copy the data */ + mpi_errno = MPIR_Localcopy(rreq->dev.user_buf, rreq->dev.user_count, + rreq->dev.datatype, + rreq->dev.real_user_buf, + rreq->dev.user_count, + rreq->dev.datatype); if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - shm_locked = 0; - } - + MPIU_ERR_POP(mpi_errno); + } goto fn_exit; } - MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN), - mpi_errno, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", op); - - /* get the function by indexing into the op table */ - uop = MPIR_OP_HDL_TO_FN(op); - - if ((op == MPI_NO_OP || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) && - MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - /* Cast away const'ness for origin_address in order to - * avoid changing the prototype for MPI_User_function */ - (*uop) ((void *) origin_addr, (char *) base + disp_unit * target_disp, - &target_count, &target_datatype); + if (HANDLE_GET_KIND(rreq->dev.op) == HANDLE_KIND_BUILTIN) + { + /* get the function by indexing into the op table */ + uop = MPIR_OP_HDL_TO_FN(rreq->dev.op); + } + else + { + /* --BEGIN ERROR HANDLING-- */ + mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", rreq->dev.op ); + return mpi_errno; + /* --END ERROR HANDLING-- */ } - else { - /* derived datatype */ + if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) + { + (*uop)(rreq->dev.user_buf, rreq->dev.real_user_buf, + &(rreq->dev.user_count), &(rreq->dev.datatype)); + } + else + { + /* derived datatype */ MPID_Segment *segp; DLOOP_VECTOR *dloop_vec; MPI_Aint first, last; - int vec_len, i, type_size, count; + int vec_len, i, count; + MPI_Aint type_size; MPI_Datatype type; - MPI_Aint true_lb, true_extent, extent; - void *tmp_buf = NULL, *target_buf; - const void *source_buf; - - if (origin_datatype != target_datatype) { - /* first copy the data into a temporary buffer with - * the same datatype as the target. Then do the - * accumulate operation. */ - - MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent); - MPID_Datatype_get_extent_macro(target_datatype, extent); - - MPIU_CHKLMEM_MALLOC(tmp_buf, void *, - target_count * (MPIR_MAX(extent, true_extent)), - mpi_errno, "temporary buffer"); - /* adjust for potential negative lower bound in datatype */ - tmp_buf = (void *) ((char *) tmp_buf - true_lb); - - mpi_errno = MPIR_Localcopy(origin_addr, origin_count, - origin_datatype, tmp_buf, target_count, target_datatype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } + MPID_Datatype *dtp; + + segp = MPID_Segment_alloc(); + /* --BEGIN ERROR HANDLING-- */ + if (!segp) + { + mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 ); + MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP); + return mpi_errno; } - - if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - /* target predefined type, origin derived datatype */ - - (*uop) (tmp_buf, (char *) base + disp_unit * target_disp, - &target_count, &target_datatype); + /* --END ERROR HANDLING-- */ + MPID_Segment_init(NULL, rreq->dev.user_count, + rreq->dev.datatype, segp, 0); + first = 0; + last = SEGMENT_IGNORE_LAST; + + MPID_Datatype_get_ptr(rreq->dev.datatype, dtp); + vec_len = dtp->max_contig_blocks * rreq->dev.user_count + 1; + /* +1 needed because Rob says so */ + dloop_vec = (DLOOP_VECTOR *) + MPIU_Malloc(vec_len * sizeof(DLOOP_VECTOR)); + /* --BEGIN ERROR HANDLING-- */ + if (!dloop_vec) + { + mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 ); + MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP); + return mpi_errno; } - else { - - segp = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER, - "**nomem", "**nomem %s", "MPID_Segment_alloc"); - MPID_Segment_init(NULL, target_count, target_datatype, segp, 0); - first = 0; - last = SEGMENT_IGNORE_LAST; - - MPID_Datatype_get_ptr(target_datatype, dtp); - vec_len = dtp->max_contig_blocks * target_count + 1; - /* +1 needed because Rob says so */ - MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *, - vec_len * sizeof(DLOOP_VECTOR), mpi_errno, "dloop vector"); - - MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len); - - source_buf = (tmp_buf != NULL) ? (const void *) tmp_buf : origin_addr; - target_buf = (char *) base + disp_unit * target_disp; - type = dtp->eltype; - type_size = MPID_Datatype_get_basic_size(type); - - for (i = 0; i < vec_len; i++) { - MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int); - (*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), - (char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), - &count, &type); - } - - MPID_Segment_free(segp); + /* --END ERROR HANDLING-- */ + + MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len); + + type = dtp->eltype; + MPID_Datatype_get_size_macro(type, type_size); + for (i=0; i<vec_len; i++) + { + MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size, int); + (*uop)((char *)rreq->dev.user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), + (char *)rreq->dev.real_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), + &count, &type); } - } - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - shm_locked = 0; + MPID_Segment_free(segp); + MPIU_Free(dloop_vec); } - fn_exit: - MPIU_CHKLMEM_FREEALL(); - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - } - goto fn_exit; - /* --END ERROR HANDLING-- */ -} + fn_exit: + /* free the temporary buffer */ + MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); + MPIU_Free((char *) rreq->dev.user_buf + true_lb); + MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP); -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Shm_get_op -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Shm_get_op(void *origin_addr, int origin_count, - MPI_Datatype origin_datatype, int target_rank, - MPI_Aint target_disp, int target_count, - MPI_Datatype target_datatype, MPID_Win * win_ptr) -{ - void *base = NULL; - int disp_unit; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_OP); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_GET_OP); - - if (win_ptr->shm_allocated == TRUE) { - base = win_ptr->shm_base_addrs[target_rank]; - disp_unit = win_ptr->disp_units[target_rank]; - } - else { - base = win_ptr->base; - disp_unit = win_ptr->disp_unit; - } - - mpi_errno = shm_copy((char *) base + disp_unit * target_disp, target_count, target_datatype, - origin_addr, origin_count, origin_datatype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_GET_OP); return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: + fn_fail: goto fn_exit; - /* --END ERROR HANDLING-- */ } - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Shm_cas_op -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Shm_cas_op(const void *origin_addr, const void *compare_addr, - void *result_addr, MPI_Datatype datatype, int target_rank, - MPI_Aint target_disp, MPID_Win * win_ptr) +static inline int wait_progress_engine(void) { - void *base = NULL, *dest_addr = NULL; - int disp_unit; - MPI_Aint len; - int shm_locked = 0; int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP); + MPID_Progress_state progress_state; - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP); - - if (win_ptr->shm_allocated == TRUE) { - base = win_ptr->shm_base_addrs[target_rank]; - disp_unit = win_ptr->disp_units[target_rank]; - - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - shm_locked = 1; - } - else { - base = win_ptr->base; - disp_unit = win_ptr->disp_unit; - } - - dest_addr = (char *) base + disp_unit * target_disp; - - MPID_Datatype_get_size_macro(datatype, len); - MPIU_Memcpy(result_addr, dest_addr, len); - - if (MPIR_Compare_equal(compare_addr, dest_addr, datatype)) { - MPIU_Memcpy(dest_addr, origin_addr, len); - } - - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - shm_locked = 0; + MPID_Progress_start(&progress_state); + mpi_errno = MPID_Progress_wait(&progress_state); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPID_Progress_end(&progress_state); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); } + /* --END ERROR HANDLING-- */ + MPID_Progress_end(&progress_state); fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_CAS_OP); return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ fn_fail: - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - } goto fn_exit; - /* --END ERROR HANDLING-- */ } - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Shm_fop_op -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_addr, - MPI_Datatype datatype, int target_rank, - MPI_Aint target_disp, MPI_Op op, MPID_Win * win_ptr) +static inline int poke_progress_engine(void) { - void *base = NULL, *dest_addr = NULL; - MPI_User_function *uop = NULL; - int disp_unit; - MPI_Aint len; - int one, shm_locked = 0; int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP); + MPID_Progress_state progress_state; - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP); - - if (win_ptr->shm_allocated == TRUE) { - base = win_ptr->shm_base_addrs[target_rank]; - disp_unit = win_ptr->disp_units[target_rank]; - - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - shm_locked = 1; - } - else { - base = win_ptr->base; - disp_unit = win_ptr->disp_unit; - } - - dest_addr = (char *) base + disp_unit * target_disp; - - MPID_Datatype_get_size_macro(datatype, len); - MPIU_Memcpy(result_addr, dest_addr, len); - - uop = MPIR_OP_HDL_TO_FN(op); - one = 1; - - (*uop) ((void *) origin_addr, dest_addr, &one, &datatype); - - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - shm_locked = 0; - } + MPID_Progress_start(&progress_state); + mpi_errno = MPID_Progress_poke(); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); + MPID_Progress_end(&progress_state); fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_FOP_OP); return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ fn_fail: - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - } goto fn_exit; - /* --END ERROR HANDLING-- */ } - -#undef FUNCNAME -#undef FCNAME - -#endif +#endif /* MPID_RMA_H_INCLUDED */ diff --git a/src/mpid/ch3/src/Makefile.mk b/src/mpid/ch3/src/Makefile.mk index 4272c26..c3b528e 100644 --- a/src/mpid/ch3/src/Makefile.mk +++ b/src/mpid/ch3/src/Makefile.mk @@ -17,10 +17,11 @@ mpi_core_sources += \ src/mpid/ch3/src/ch3u_port.c \ src/mpid/ch3/src/ch3u_recvq.c \ src/mpid/ch3/src/ch3u_request.c \ - src/mpid/ch3/src/ch3u_rma_acc_ops.c \ + src/mpid/ch3/src/ch3u_rma_oplist.c \ src/mpid/ch3/src/ch3u_rma_ops.c \ src/mpid/ch3/src/ch3u_rma_reqops.c \ src/mpid/ch3/src/ch3u_rma_sync.c \ + src/mpid/ch3/src/ch3u_rma_pkthandler.c \ src/mpid/ch3/src/ch3u_rndv.c \ src/mpid/ch3/src/ch3u_eager.c \ src/mpid/ch3/src/ch3u_eagersync.c \ @@ -56,6 +57,7 @@ mpi_core_sources += \ src/mpid/ch3/src/mpid_startall.c \ src/mpid/ch3/src/mpid_vc.c \ src/mpid/ch3/src/mpid_rma.c \ + src/mpid/ch3/src/mpidi_rma.c \ src/mpid/ch3/src/mpidi_isend_self.c \ src/mpid/ch3/src/mpidi_pg.c \ src/mpid/ch3/src/mpidi_printf.c diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index db6760c..5c1b952 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -8,7 +8,6 @@ #include "mpidrma.h" static int create_derived_datatype(MPID_Request * rreq, MPID_Datatype ** dtp); -static int do_accumulate_op(MPID_Request * rreq); static int do_simple_accumulate(MPIDI_PT_single_op *single_op); static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue); @@ -827,119 +826,6 @@ static int create_derived_datatype(MPID_Request *req, MPID_Datatype **dtp) return mpi_errno; } - -#undef FUNCNAME -#define FUNCNAME do_accumulate_op -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int do_accumulate_op(MPID_Request *rreq) -{ - int mpi_errno = MPI_SUCCESS; - MPI_Aint true_lb, true_extent; - MPI_User_function *uop; - MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP); - - MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP); - - if (rreq->dev.op == MPI_REPLACE) - { - /* simply copy the data */ - mpi_errno = MPIR_Localcopy(rreq->dev.user_buf, rreq->dev.user_count, - rreq->dev.datatype, - rreq->dev.real_user_buf, - rreq->dev.user_count, - rreq->dev.datatype); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - goto fn_exit; - } - - if (HANDLE_GET_KIND(rreq->dev.op) == HANDLE_KIND_BUILTIN) - { - /* get the function by indexing into the op table */ - uop = MPIR_OP_HDL_TO_FN(rreq->dev.op); - } - else - { - /* --BEGIN ERROR HANDLING-- */ - mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", rreq->dev.op ); - return mpi_errno; - /* --END ERROR HANDLING-- */ - } - - if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) - { - (*uop)(rreq->dev.user_buf, rreq->dev.real_user_buf, - &(rreq->dev.user_count), &(rreq->dev.datatype)); - } - else - { - /* derived datatype */ - MPID_Segment *segp; - DLOOP_VECTOR *dloop_vec; - MPI_Aint first, last; - int vec_len, i, count; - MPI_Aint type_size; - MPI_Datatype type; - MPID_Datatype *dtp; - - segp = MPID_Segment_alloc(); - /* --BEGIN ERROR HANDLING-- */ - if (!segp) - { - mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 ); - MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP); - return mpi_errno; - } - /* --END ERROR HANDLING-- */ - MPID_Segment_init(NULL, rreq->dev.user_count, - rreq->dev.datatype, segp, 0); - first = 0; - last = SEGMENT_IGNORE_LAST; - - MPID_Datatype_get_ptr(rreq->dev.datatype, dtp); - vec_len = dtp->max_contig_blocks * rreq->dev.user_count + 1; - /* +1 needed because Rob says so */ - dloop_vec = (DLOOP_VECTOR *) - MPIU_Malloc(vec_len * sizeof(DLOOP_VECTOR)); - /* --BEGIN ERROR HANDLING-- */ - if (!dloop_vec) - { - mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 ); - MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP); - return mpi_errno; - } - /* --END ERROR HANDLING-- */ - - MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len); - - type = dtp->eltype; - MPID_Datatype_get_size_macro(type, type_size); - for (i=0; i<vec_len; i++) - { - MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size, int); - (*uop)((char *)rreq->dev.user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), - (char *)rreq->dev.real_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF), - &count, &type); - } - - MPID_Segment_free(segp); - MPIU_Free(dloop_vec); - } - - fn_exit: - /* free the temporary buffer */ - MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); - MPIU_Free((char *) rreq->dev.user_buf + true_lb); - - MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP); - - return mpi_errno; - fn_fail: - goto fn_exit; -} - static int entered_flag = 0; static int entered_count = 0; @@ -1129,43 +1015,6 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) } -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPID_Win *win_ptr, - MPI_Win source_win_handle) -{ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done; - MPID_Request *req; - int mpi_errno=MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); - - MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE); - pt_rma_done_pkt->source_win_handle = source_win_handle; - pt_rma_done_pkt->target_rank = win_ptr->comm_ptr->rank; - - /* Because this is in a packet handler, it is already within a critical section */ - /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */ - mpi_errno = MPIDI_CH3_iStartMsg(vc, pt_rma_done_pkt, sizeof(*pt_rma_done_pkt), &req); - /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */ - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg"); - } - - if (req != NULL) - { - MPID_Request_release(req); - } - - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT); - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME do_simple_accumulate diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c new file mode 100644 index 0000000..b8b113f --- /dev/null +++ b/src/mpid/ch3/src/ch3u_rma_oplist.c @@ -0,0 +1,932 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#include "mpidimpl.h" +#include "mpidrma.h" + +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_CH3_RMA_ACC_IMMED + category : CH3 + type : boolean + default : true + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Use the immediate accumulate optimization + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + +static int send_flush_msg(int dest, MPID_Win *win_ptr); +static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags); +static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, + MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags); +static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags); +static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, + MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags); + +/* create_datatype() creates a new struct datatype for the dtype_info + and the dataloop of the target datatype together with the user data */ +#undef FUNCNAME +#define FUNCNAME create_datatype +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, + const void *dataloop, MPI_Aint dataloop_sz, + const void *o_addr, int o_count, MPI_Datatype o_datatype, + MPID_Datatype ** combined_dtp) +{ + int mpi_errno = MPI_SUCCESS; + /* datatype_set_contents wants an array 'ints' which is the + * blocklens array with count prepended to it. So blocklens + * points to the 2nd element of ints to avoid having to copy + * blocklens into ints later. */ + int ints[4]; + int *blocklens = &ints[1]; + MPI_Aint displaces[3]; + MPI_Datatype datatypes[3]; + const int count = 3; + MPI_Datatype combined_datatype; + MPIDI_STATE_DECL(MPID_STATE_CREATE_DATATYPE); + + MPIDI_FUNC_ENTER(MPID_STATE_CREATE_DATATYPE); + + /* create datatype */ + displaces[0] = MPIU_PtrToAint(dtype_info); + blocklens[0] = sizeof(*dtype_info); + datatypes[0] = MPI_BYTE; + + displaces[1] = MPIU_PtrToAint(dataloop); + MPIU_Assign_trunc(blocklens[1], dataloop_sz, int); + datatypes[1] = MPI_BYTE; + + displaces[2] = MPIU_PtrToAint(o_addr); + blocklens[2] = o_count; + datatypes[2] = o_datatype; + + mpi_errno = MPID_Type_struct(count, blocklens, displaces, datatypes, &combined_datatype); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + + ints[0] = count; + + MPID_Datatype_get_ptr(combined_datatype, *combined_dtp); + mpi_errno = MPID_Datatype_set_contents(*combined_dtp, MPI_COMBINER_STRUCT, count + 1, /* ints (cnt,blklen) */ + count, /* aints (disps) */ + count, /* types */ + ints, displaces, datatypes); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + + /* Commit datatype */ + + MPID_Dataloop_create(combined_datatype, + &(*combined_dtp)->dataloop, + &(*combined_dtp)->dataloop_size, + &(*combined_dtp)->dataloop_depth, MPID_DATALOOP_HOMOGENEOUS); + + /* create heterogeneous dataloop */ + MPID_Dataloop_create(combined_datatype, + &(*combined_dtp)->hetero_dloop, + &(*combined_dtp)->hetero_dloop_size, + &(*combined_dtp)->hetero_dloop_depth, MPID_DATALOOP_HETEROGENEOUS); + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_CREATE_DATATYPE); + return mpi_errno; + fn_fail: + goto fn_exit; +} + + +#undef FUNCNAME +#define FUNCNAME send_rma_msg +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, + MPIDI_CH3_Pkt_flags_t flags, + MPI_Win source_win_handle, + MPI_Win target_win_handle, + MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request) +{ + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_put_t *put_pkt = &upkt.put; + MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum; + MPID_IOV iov[MPID_IOV_LIMIT]; + int mpi_errno = MPI_SUCCESS; + int origin_dt_derived, target_dt_derived, iovcnt; + MPI_Aint origin_type_size; + MPIDI_VC_t *vc; + MPID_Comm *comm_ptr; + MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL; + MPID_Request *resp_req = NULL; + MPIU_CHKPMEM_DECL(1); + MPIDI_STATE_DECL(MPID_STATE_SEND_RMA_MSG); + MPIDI_STATE_DECL(MPID_STATE_MEMCPY); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_RMA_MSG); + + *request = NULL; + + if (rma_op->type == MPIDI_RMA_PUT) { + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + put_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + put_pkt->flags = flags; + put_pkt->count = rma_op->target_count; + put_pkt->datatype = rma_op->target_datatype; + put_pkt->dataloop_size = 0; + put_pkt->target_win_handle = target_win_handle; + put_pkt->source_win_handle = source_win_handle; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt; + iov[0].MPID_IOV_LEN = sizeof(*put_pkt); + } + else if (rma_op->type == MPIDI_RMA_GET_ACCUMULATE) { + /* Create a request for the GACC response. Store the response buf, count, and + * datatype in it, and pass the request's handle in the GACC packet. When the + * response comes from the target, it will contain the request handle. */ + resp_req = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + + MPIU_Object_set_ref(resp_req, 2); + + resp_req->dev.user_buf = rma_op->result_addr; + resp_req->dev.user_count = rma_op->result_count; + resp_req->dev.datatype = rma_op->result_datatype; + resp_req->dev.target_win_handle = target_win_handle; + resp_req->dev.source_win_handle = source_win_handle; + + if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { + MPID_Datatype *result_dtp = NULL; + MPID_Datatype_get_ptr(resp_req->dev.datatype, result_dtp); + resp_req->dev.datatype_ptr = result_dtp; + /* this will cause the datatype to be freed when the + * request is freed. */ + } + + /* Note: Get_accumulate uses the same packet type as accumulate */ + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + accum_pkt->flags = flags; + accum_pkt->count = rma_op->target_count; + accum_pkt->datatype = rma_op->target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = rma_op->op; + accum_pkt->target_win_handle = target_win_handle; + accum_pkt->source_win_handle = source_win_handle; + accum_pkt->request_handle = resp_req->handle; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; + iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); + } + else { + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + accum_pkt->flags = flags; + accum_pkt->count = rma_op->target_count; + accum_pkt->datatype = rma_op->target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = rma_op->op; + accum_pkt->target_win_handle = target_win_handle; + accum_pkt->source_win_handle = source_win_handle; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; + iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); + } + + /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, + * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); + * fflush(stdout); + */ + + comm_ptr = win_ptr->comm_ptr; + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + + if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) { + origin_dt_derived = 1; + MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp); + } + else { + origin_dt_derived = 0; + } + + if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) { + target_dt_derived = 1; + MPID_Datatype_get_ptr(rma_op->target_datatype, target_dtp); + } + else { + target_dt_derived = 0; + } + + if (target_dt_derived) { + /* derived datatype on target. fill derived datatype info */ + dtype_info->is_contig = target_dtp->is_contig; + dtype_info->max_contig_blocks = target_dtp->max_contig_blocks; + dtype_info->size = target_dtp->size; + dtype_info->extent = target_dtp->extent; + dtype_info->dataloop_size = target_dtp->dataloop_size; + dtype_info->dataloop_depth = target_dtp->dataloop_depth; + dtype_info->eltype = target_dtp->eltype; + dtype_info->dataloop = target_dtp->dataloop; + dtype_info->ub = target_dtp->ub; + dtype_info->lb = target_dtp->lb; + dtype_info->true_ub = target_dtp->true_ub; + dtype_info->true_lb = target_dtp->true_lb; + dtype_info->has_sticky_ub = target_dtp->has_sticky_ub; + dtype_info->has_sticky_lb = target_dtp->has_sticky_lb; + + MPIU_CHKPMEM_MALLOC(*dataloop, void *, target_dtp->dataloop_size, mpi_errno, "dataloop"); + + MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY); + MPIU_Memcpy(*dataloop, target_dtp->dataloop, target_dtp->dataloop_size); + MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY); + /* the dataloop can have undefined padding sections, so we need to let + * valgrind know that it is OK to pass this data to writev later on */ + MPL_VG_MAKE_MEM_DEFINED(*dataloop, target_dtp->dataloop_size); + + if (rma_op->type == MPIDI_RMA_PUT) { + put_pkt->dataloop_size = target_dtp->dataloop_size; + } + else { + accum_pkt->dataloop_size = target_dtp->dataloop_size; + } + } + + MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); + + if (!target_dt_derived) { + /* basic datatype on target */ + if (!origin_dt_derived) { + /* basic datatype on origin */ + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr; + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; + iovcnt = 2; + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + else { + /* derived datatype on origin */ + *request = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(*request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + + MPIU_Object_set_ref(*request, 2); + (*request)->kind = MPID_REQUEST_SEND; + + (*request)->dev.segment_ptr = MPID_Segment_alloc(); + MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, + "**nomem", "**nomem %s", "MPID_Segment_alloc"); + + (*request)->dev.datatype_ptr = origin_dtp; + /* this will cause the datatype to be freed when the request + * is freed. */ + MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count, + rma_op->origin_datatype, (*request)->dev.segment_ptr, 0); + (*request)->dev.segment_first = 0; + (*request)->dev.segment_size = rma_op->origin_count * origin_type_size; + + (*request)->dev.OnFinal = 0; + (*request)->dev.OnDataAvail = 0; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = + vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + } + else { + /* derived datatype on target */ + MPID_Datatype *combined_dtp = NULL; + + *request = MPID_Request_create(); + if (*request == NULL) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + } + + MPIU_Object_set_ref(*request, 2); + (*request)->kind = MPID_REQUEST_SEND; + + (*request)->dev.segment_ptr = MPID_Segment_alloc(); + MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, + "**nomem", "**nomem %s", "MPID_Segment_alloc"); + + /* create a new datatype containing the dtype_info, dataloop, and origin data */ + + mpi_errno = + create_datatype(dtype_info, *dataloop, target_dtp->dataloop_size, rma_op->origin_addr, + rma_op->origin_count, rma_op->origin_datatype, &combined_dtp); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + + (*request)->dev.datatype_ptr = combined_dtp; + /* combined_datatype will be freed when request is freed */ + + MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, (*request)->dev.segment_ptr, 0); + (*request)->dev.segment_first = 0; + (*request)->dev.segment_size = combined_dtp->size; + + (*request)->dev.OnFinal = 0; + (*request)->dev.OnDataAvail = 0; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + /* we're done with the datatypes */ + if (origin_dt_derived) + MPID_Datatype_release(origin_dtp); + MPID_Datatype_release(target_dtp); + } + + /* This operation can generate two requests; one for inbound and one for + * outbound data. */ + if (resp_req != NULL) { + if (*request != NULL) { + /* If we have both inbound and outbound requests (i.e. GACC + * operation), we need to ensure that the source buffer is + * available and that the response data has been received before + * informing the origin that this operation is complete. Because + * the update needs to be done atomically at the target, they will + * not send back data until it has been received. Therefore, + * completion of the response request implies that the send request + * has completed. + * + * Therefore: refs on the response request are set to two: one is + * held by the progress engine and the other by the RMA op + * completion code. Refs on the outbound request are set to one; + * it will be completed by the progress engine. + */ + + MPID_Request_release(*request); + *request = resp_req; + + } + else { + *request = resp_req; + } + + /* For error checking */ + resp_req = NULL; + } + + fn_exit: + MPIU_CHKPMEM_COMMIT(); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_RMA_MSG); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + if (resp_req) { + MPID_Request_release(resp_req); + } + if (*request) { + MPIU_CHKPMEM_REAP(); + if ((*request)->dev.datatype_ptr) + MPID_Datatype_release((*request)->dev.datatype_ptr); + MPID_Request_release(*request); + } + *request = NULL; + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + +/* + * Use this for contiguous accumulate operations + */ +#undef FUNCNAME +#define FUNCNAME send_contig_acc_msg +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, + MPID_Win * win_ptr, + MPIDI_CH3_Pkt_flags_t flags, + MPI_Win source_win_handle, + MPI_Win target_win_handle, MPID_Request ** request) +{ + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum; + MPID_IOV iov[MPID_IOV_LIMIT]; + int mpi_errno = MPI_SUCCESS; + int iovcnt; + MPI_Aint origin_type_size; + MPIDI_VC_t *vc; + MPID_Comm *comm_ptr; + size_t len; + MPIDI_STATE_DECL(MPID_STATE_SEND_CONTIG_ACC_MSG); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_CONTIG_ACC_MSG); + + *request = NULL; + + MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); + /* FIXME: Make this size check efficient and match the packet type */ + MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); + if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) { + MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &upkt.accum_immed; + void *dest = accumi_pkt->data, *src = rma_op->origin_addr; + + MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED); + accumi_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + accumi_pkt->flags = flags; + accumi_pkt->count = rma_op->target_count; + accumi_pkt->datatype = rma_op->target_datatype; + accumi_pkt->op = rma_op->op; + accumi_pkt->target_win_handle = target_win_handle; + accumi_pkt->source_win_handle = source_win_handle; + + switch (len) { + case 1: + *(uint8_t *) dest = *(uint8_t *) src; + break; + case 2: + *(uint16_t *) dest = *(uint16_t *) src; + break; + case 4: + *(uint32_t *) dest = *(uint32_t *) src; + break; + case 8: + *(uint64_t *) dest = *(uint64_t *) src; + break; + default: + MPIU_Memcpy(accumi_pkt->data, (void *) rma_op->origin_addr, len); + } + comm_ptr = win_ptr->comm_ptr; + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), request); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + goto fn_exit; + } + + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + accum_pkt->flags = flags; + accum_pkt->count = rma_op->target_count; + accum_pkt->datatype = rma_op->target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = rma_op->op; + accum_pkt->target_win_handle = target_win_handle; + accum_pkt->source_win_handle = source_win_handle; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; + iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); + + /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, + * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); + * fflush(stdout); + */ + + comm_ptr = win_ptr->comm_ptr; + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + + + /* basic datatype on target */ + /* basic datatype on origin */ + /* FIXME: This is still very heavyweight for a small message operation, + * such as a single word update */ + /* One possibility is to use iStartMsg with a buffer that is just large + * enough, though note that nemesis has an optimization for this */ + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr; + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; + iovcnt = 2; + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_CONTIG_ACC_MSG); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + if (*request) { + MPID_Request_release(*request); + } + *request = NULL; + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +/* + * Initiate an immediate RMW accumulate operation + */ +#undef FUNCNAME +#define FUNCNAME send_immed_rmw_msg +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, + MPID_Win * win_ptr, + MPIDI_CH3_Pkt_flags_t flags, + MPI_Win source_win_handle, + MPI_Win target_win_handle, MPID_Request ** request) +{ + int mpi_errno = MPI_SUCCESS; + MPID_Request *rmw_req = NULL, *resp_req = NULL; + MPIDI_VC_t *vc; + MPID_Comm *comm_ptr; + MPI_Aint len; + MPIDI_STATE_DECL(MPID_STATE_SEND_IMMED_RMW_MSG); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_IMMED_RMW_MSG); + + *request = NULL; + + /* Create a request for the RMW response. Store the origin buf, count, and + * datatype in it, and pass the request's handle RMW packet. When the + * response comes from the target, it will contain the request handle. */ + resp_req = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + *request = resp_req; + + /* Set refs on the request to 2: one for the response message, and one for + * the partial completion handler */ + MPIU_Object_set_ref(resp_req, 2); + + resp_req->dev.user_buf = rma_op->result_addr; + resp_req->dev.user_count = rma_op->result_count; + resp_req->dev.datatype = rma_op->result_datatype; + resp_req->dev.target_win_handle = target_win_handle; + resp_req->dev.source_win_handle = source_win_handle; + + /* REQUIRE: All datatype arguments must be of the same, builtin + * type and counts must be 1. */ + MPID_Datatype_get_size_macro(rma_op->origin_datatype, len); + comm_ptr = win_ptr->comm_ptr; + + if (rma_op->type == MPIDI_RMA_COMPARE_AND_SWAP) { + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_cas_t *cas_pkt = &upkt.cas; + + MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); + + MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + + cas_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + cas_pkt->flags = flags; + cas_pkt->datatype = rma_op->target_datatype; + cas_pkt->target_win_handle = target_win_handle; + cas_pkt->request_handle = resp_req->handle; + + MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); + MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); + + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_pkt, sizeof(*cas_pkt), &rmw_req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + if (rmw_req != NULL) { + MPID_Request_release(rmw_req); + } + } + + else if (rma_op->type == MPIDI_RMA_FETCH_AND_OP) { + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_fop_t *fop_pkt = &upkt.fop; + + MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u)); + + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + + fop_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + fop_pkt->flags = flags; + fop_pkt->datatype = rma_op->target_datatype; + fop_pkt->target_win_handle = target_win_handle; + fop_pkt->request_handle = resp_req->handle; + fop_pkt->op = rma_op->op; + + if (len <= sizeof(fop_pkt->origin_data) || rma_op->op == MPI_NO_OP) { + /* Embed FOP data in the packet header */ + if (rma_op->op != MPI_NO_OP) { + MPIU_Memcpy(fop_pkt->origin_data, rma_op->origin_addr, len); + } + + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_pkt, sizeof(*fop_pkt), &rmw_req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + if (rmw_req != NULL) { + MPID_Request_release(rmw_req); + } + } + else { + /* Data is too big to copy into the FOP header, use an IOV to send it */ + MPID_IOV iov[MPID_IOV_LIMIT]; + + rmw_req = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(rmw_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + MPIU_Object_set_ref(rmw_req, 1); + + rmw_req->dev.OnFinal = 0; + rmw_req->dev.OnDataAvail = 0; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr; + iov[1].MPID_IOV_LEN = len; /* count == 1 */ + + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iSendv(vc, rmw_req, iov, 2); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + } + else { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_IMMED_RMW_MSG); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + if (*request) { + MPID_Request_release(*request); + } + *request = NULL; + if (rmw_req) { + MPID_Request_release(rmw_req); + } + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + + +#undef FUNCNAME +#define FUNCNAME recv_rma_msg +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, + MPIDI_CH3_Pkt_flags_t flags, + MPI_Win source_win_handle, + MPI_Win target_win_handle, + MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request) +{ + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_get_t *get_pkt = &upkt.get; + int mpi_errno = MPI_SUCCESS; + MPIDI_VC_t *vc; + MPID_Comm *comm_ptr; + MPID_Request *req = NULL; + MPID_Datatype *dtp; + MPID_IOV iov[MPID_IOV_LIMIT]; + MPIU_CHKPMEM_DECL(1); + MPIDI_STATE_DECL(MPID_STATE_RECV_RMA_MSG); + MPIDI_STATE_DECL(MPID_STATE_MEMCPY); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_RECV_RMA_MSG); + + /* create a request, store the origin buf, cnt, datatype in it, + * and pass a handle to it in the get packet. When the get + * response comes from the target, it will contain the request + * handle. */ + req = MPID_Request_create(); + if (req == NULL) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + } + + *request = req; + + MPIU_Object_set_ref(req, 2); + + req->dev.user_buf = rma_op->origin_addr; + req->dev.user_count = rma_op->origin_count; + req->dev.datatype = rma_op->origin_datatype; + req->dev.target_win_handle = MPI_WIN_NULL; + req->dev.source_win_handle = source_win_handle; + if (!MPIR_DATATYPE_IS_PREDEFINED(req->dev.datatype)) { + MPID_Datatype_get_ptr(req->dev.datatype, dtp); + req->dev.datatype_ptr = dtp; + /* this will cause the datatype to be freed when the + * request is freed. */ + } + + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + + win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; + get_pkt->flags = flags; + get_pkt->count = rma_op->target_count; + get_pkt->datatype = rma_op->target_datatype; + get_pkt->request_handle = req->handle; + get_pkt->target_win_handle = target_win_handle; + get_pkt->source_win_handle = source_win_handle; + +/* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, + rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); + fflush(stdout); +*/ + + comm_ptr = win_ptr->comm_ptr; + MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + + if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) { + /* basic datatype on target. simply send the get_pkt. */ + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, get_pkt, sizeof(*get_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + } + else { + /* derived datatype on target. fill derived datatype info and + * send it along with get_pkt. */ + + MPID_Datatype_get_ptr(rma_op->target_datatype, dtp); + dtype_info->is_contig = dtp->is_contig; + dtype_info->max_contig_blocks = dtp->max_contig_blocks; + dtype_info->size = dtp->size; + dtype_info->extent = dtp->extent; + dtype_info->dataloop_size = dtp->dataloop_size; + dtype_info->dataloop_depth = dtp->dataloop_depth; + dtype_info->eltype = dtp->eltype; + dtype_info->dataloop = dtp->dataloop; + dtype_info->ub = dtp->ub; + dtype_info->lb = dtp->lb; + dtype_info->true_ub = dtp->true_ub; + dtype_info->true_lb = dtp->true_lb; + dtype_info->has_sticky_ub = dtp->has_sticky_ub; + dtype_info->has_sticky_lb = dtp->has_sticky_lb; + + MPIU_CHKPMEM_MALLOC(*dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop"); + + MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY); + MPIU_Memcpy(*dataloop, dtp->dataloop, dtp->dataloop_size); + MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY); + + /* the dataloop can have undefined padding sections, so we need to let + * valgrind know that it is OK to pass this data to writev later on */ + MPL_VG_MAKE_MEM_DEFINED(*dataloop, dtp->dataloop_size); + + get_pkt->dataloop_size = dtp->dataloop_size; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) dtype_info; + iov[1].MPID_IOV_LEN = sizeof(*dtype_info); + iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) * dataloop; + iov[2].MPID_IOV_LEN = dtp->dataloop_size; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, 3, &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + + /* release the target datatype */ + MPID_Datatype_release(dtp); + } + + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + + /* release the request returned by iStartMsg or iStartMsgv */ + if (req != NULL) { + MPID_Request_release(req); + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_RECV_RMA_MSG); + return mpi_errno; + + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME send_flush_msg +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int send_flush_msg(int dest, MPID_Win * win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush; + MPID_Request *req = NULL; + MPIDI_VC_t *vc; + MPIDI_STATE_DECL(MPID_STATE_SEND_FLUSH_MSG); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_FLUSH_MSG); + + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); + + MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH); + flush_pkt->target_win_handle = win_ptr->all_win_handles[dest]; + flush_pkt->source_win_handle = win_ptr->handle; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); + + /* Release the request returned by iStartMsg */ + if (req != NULL) { + MPID_Request_release(req); + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_FLUSH_MSG); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +int MPIDI_CH3I_Issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, + MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle, + MPI_Win target_win_handle) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP); + + switch (op_ptr->type) { + case (MPIDI_RMA_PUT): + case (MPIDI_RMA_ACCUMULATE): + mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle, + target_win_handle, &op_ptr->dtype_info, + &op_ptr->dataloop, &op_ptr->request); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + break; + case (MPIDI_RMA_GET_ACCUMULATE): + if (op_ptr->op == MPI_NO_OP) { + /* Note: Origin arguments are ignored for NO_OP, so we don't + * need to release a ref to the origin datatype. */ + + /* Convert the GAcc to a Get */ + op_ptr->type = MPIDI_RMA_GET; + op_ptr->origin_addr = op_ptr->result_addr; + op_ptr->origin_count = op_ptr->result_count; + op_ptr->origin_datatype = op_ptr->result_datatype; + + mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags, source_win_handle, + target_win_handle, &op_ptr->dtype_info, + &op_ptr->dataloop, &op_ptr->request); + } + else { + mpi_errno = send_rma_msg(op_ptr, win_ptr, flags, source_win_handle, + target_win_handle, &op_ptr->dtype_info, + &op_ptr->dataloop, &op_ptr->request); + } + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + break; + case (MPIDI_RMA_ACC_CONTIG): + mpi_errno = send_contig_acc_msg(op_ptr, win_ptr, flags, + source_win_handle, target_win_handle, &op_ptr->request); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + break; + case (MPIDI_RMA_GET): + mpi_errno = recv_rma_msg(op_ptr, win_ptr, flags, + source_win_handle, target_win_handle, + &op_ptr->dtype_info, &op_ptr->dataloop, &op_ptr->request); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + break; + case (MPIDI_RMA_COMPARE_AND_SWAP): + case (MPIDI_RMA_FETCH_AND_OP): + mpi_errno = send_immed_rmw_msg(op_ptr, win_ptr, flags, + source_win_handle, target_win_handle, &op_ptr->request); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + break; + default: + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winInvalidOp"); + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_ISSUE_RMA_OP); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index d91a626..9024726 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -11,107 +11,6 @@ static int enableShortACC = 1; #define MPIDI_PASSIVE_TARGET_DONE_TAG 348297 #define MPIDI_PASSIVE_TARGET_RMA_TAG 563924 -/* - * TODO: - * Explore use of alternate allocation mechanisms for the RMA queue elements - * (Because profiling has shown that queue element allocation/deallocation - * can take a significant amount of time in the RMA operations). - * 1: Current approach (uses perm memory malloc/free) - * 2: Preallocate and maintain list (use perm memory malloc, but - * free onto window; use first; free on window free) - * 3: Preallocate and maintain list (use separate memory, but free to - * thread/process; free in Finalize handler. Option to use for - * single-threaded to avoid thread overheads) - * Possible interface - * int MPIDI_RMAListAlloc(MPIDI_RMA_Op_t **a,MPID_Win *win) - * int MPIDI_RMAListFree(MPIDI_RMA_Op_t *a, MPID_Win *win) - * return value is error code (e.g., allocation failure). - */ - -#undef FUNCNAME -#define FUNCNAME MPIDI_Win_free -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_Win_free(MPID_Win ** win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - int in_use; - MPID_Comm *comm_ptr; - int errflag = FALSE; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FREE); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FREE); - - MPIU_ERR_CHKANDJUMP((*win_ptr)->epoch_state != MPIDI_EPOCH_NONE, - mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - - if (!(*win_ptr)->shm_allocated) { - /* when SHM is allocated, we already did a global barrier in - MPIDI_CH3_SHM_Win_free, so we do not need to do it again here. */ - mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - } - - comm_ptr = (*win_ptr)->comm_ptr; - mpi_errno = MPIR_Comm_free_impl(comm_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIU_Free((*win_ptr)->targets); - MPIU_Free((*win_ptr)->base_addrs); - MPIU_Free((*win_ptr)->sizes); - MPIU_Free((*win_ptr)->disp_units); - MPIU_Free((*win_ptr)->all_win_handles); - - /* Free the attached buffer for windows created with MPI_Win_allocate() */ - if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE || - (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED) { - if ((*win_ptr)->shm_allocated == FALSE && (*win_ptr)->size > 0) { - MPIU_Free((*win_ptr)->base); - } - } - - MPIU_Object_release_ref(*win_ptr, &in_use); - /* MPI windows don't have reference count semantics, so this should always be true */ - MPIU_Assert(!in_use); - MPIU_Handle_obj_free(&MPID_Win_mem, *win_ptr); - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FREE); - return mpi_errno; - - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_Win_shared_query -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_Win_shared_query(MPID_Win * win_ptr, int target_rank, MPI_Aint * size, - int *disp_unit, void *baseptr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SHARED_QUERY); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SHARED_QUERY); - - *(void **) baseptr = win_ptr->base; - *size = win_ptr->size; - *disp_unit = win_ptr->disp_unit; - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SHARED_QUERY); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - #undef FUNCNAME #define FUNCNAME MPIDI_Put #undef FCNAME @@ -448,36 +347,300 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype #undef FUNCNAME -#define FUNCNAME MPIDI_Alloc_mem +#define FUNCNAME MPIDI_Get_accumulate +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, void *result_addr, int result_count, + MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, + int target_count, MPI_Datatype target_datatype, MPI_Op op, + MPID_Win * win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_msg_sz_t data_sz; + int rank; + int dt_contig ATTRIBUTE((unused)); + MPI_Aint dt_true_lb ATTRIBUTE((unused)); + MPID_Datatype *dtp; + MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE); + + if (target_rank == MPI_PROC_NULL) { + goto fn_exit; + } + + if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) { + win_ptr->epoch_state = MPIDI_EPOCH_FENCE; + } + + MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE, + mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); + + MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb); + + if (data_sz == 0) { + goto fn_exit; + } + + rank = win_ptr->comm_ptr->rank; + + if (win_ptr->shm_allocated == TRUE && target_rank != rank && + win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) { + /* check if target is local and shared memory is allocated on window, + * if so, we directly perform this operation on shared memory region. */ + + /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on + * the same node. However, in ch3:sock, even if origin and target are on the same node, they do + * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, + * which is only set to TRUE when SHM region is allocated in nemesis. + * In future we need to figure out a way to check if origin and target are in the same "SHM comm". + */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + } + + /* Do =! rank first (most likely branch?) */ + if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || + (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { + mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype, + result_addr, result_count, result_datatype, + target_rank, target_disp, target_count, + target_datatype, op, win_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + } + else { + MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Op_t *new_ptr = NULL; + + /* Append the operation to the window's RMA ops queue */ + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */ + + new_ptr->type = MPIDI_RMA_GET_ACCUMULATE; + /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t + * contain both PUT and GET like ops */ + new_ptr->origin_addr = (void *) origin_addr; + new_ptr->origin_count = origin_count; + new_ptr->origin_datatype = origin_datatype; + new_ptr->result_addr = result_addr; + new_ptr->result_count = result_count; + new_ptr->result_datatype = result_datatype; + new_ptr->target_rank = target_rank; + new_ptr->target_disp = target_disp; + new_ptr->target_count = target_count; + new_ptr->target_datatype = target_datatype; + new_ptr->op = op; + + /* if source or target datatypes are derived, increment their + * reference counts */ + if (op != MPI_NO_OP && !MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { + MPID_Datatype_get_ptr(origin_datatype, dtp); + MPID_Datatype_add_ref(dtp); + } + if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { + MPID_Datatype_get_ptr(result_datatype, dtp); + MPID_Datatype_add_ref(dtp); + } + if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { + MPID_Datatype_get_ptr(target_datatype, dtp); + MPID_Datatype_add_ref(dtp); + } + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET_ACCUMULATE); + return mpi_errno; + + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_Compare_and_swap #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -void *MPIDI_Alloc_mem(size_t size, MPID_Info * info_ptr) +int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, + void *result_addr, MPI_Datatype datatype, int target_rank, + MPI_Aint target_disp, MPID_Win * win_ptr) { - void *ap; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_ALLOC_MEM); + int mpi_errno = MPI_SUCCESS; + int rank; + MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL; + + MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMPARE_AND_SWAP); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_COMPARE_AND_SWAP); + + if (target_rank == MPI_PROC_NULL) { + goto fn_exit; + } + + if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) { + win_ptr->epoch_state = MPIDI_EPOCH_FENCE; + } - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_ALLOC_MEM); + MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE, + mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - ap = MPIU_Malloc(size); + rank = win_ptr->comm_ptr->rank; - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_ALLOC_MEM); - return ap; + if (win_ptr->shm_allocated == TRUE && target_rank != rank && + win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) { + /* check if target is local and shared memory is allocated on window, + * if so, we directly perform this operation on shared memory region. */ + + /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on + * the same node. However, in ch3:sock, even if origin and target are on the same node, they do + * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, + * which is only set to TRUE when SHM region is allocated in nemesis. + * In future we need to figure out a way to check if origin and target are in the same "SHM comm". + */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + } + + /* The datatype must be predefined, and one of: C integer, Fortran integer, + * Logical, Multi-language types, or Byte. This is checked above the ADI, + * so there's no need to check it again here. */ + + /* FIXME: For shared memory windows, we should provide an implementation + * that uses a processor atomic operation. */ + if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || + (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { + mpi_errno = MPIDI_CH3I_Shm_cas_op(origin_addr, compare_addr, result_addr, + datatype, target_rank, target_disp, win_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + } + else { + MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Op_t *new_ptr = NULL; + + /* Append this operation to the RMA ops queue */ + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP; + new_ptr->origin_addr = (void *) origin_addr; + new_ptr->origin_count = 1; + new_ptr->origin_datatype = datatype; + new_ptr->target_rank = target_rank; + new_ptr->target_disp = target_disp; + new_ptr->target_count = 1; + new_ptr->target_datatype = datatype; + new_ptr->result_addr = result_addr; + new_ptr->result_count = 1; + new_ptr->result_datatype = datatype; + new_ptr->compare_addr = (void *) compare_addr; + new_ptr->compare_count = 1; + new_ptr->compare_datatype = datatype; + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_COMPARE_AND_SWAP); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ } #undef FUNCNAME -#define FUNCNAME MPIDI_Free_mem +#define FUNCNAME MPIDI_Fetch_and_op #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_Free_mem(void *ptr) +int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, + MPI_Datatype datatype, int target_rank, + MPI_Aint target_disp, MPI_Op op, MPID_Win * win_ptr) { int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_FREE_MEM); + int rank; + MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL; + + MPIDI_STATE_DECL(MPID_STATE_MPIDI_FETCH_AND_OP); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_FETCH_AND_OP); + + if (target_rank == MPI_PROC_NULL) { + goto fn_exit; + } + + if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) { + win_ptr->epoch_state = MPIDI_EPOCH_FENCE; + } + + MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE, + mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); + + rank = win_ptr->comm_ptr->rank; + + if (win_ptr->shm_allocated == TRUE && target_rank != rank && + win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) { + /* check if target is local and shared memory is allocated on window, + * if so, we directly perform this operation on shared memory region. */ + + /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on + * the same node. However, in ch3:sock, even if origin and target are on the same node, they do + * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, + * which is only set to TRUE when SHM region is allocated in nemesis. + * In future we need to figure out a way to check if origin and target are in the same "SHM comm". + */ + MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); + MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); + } - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_FREE_MEM); + /* The datatype and op must be predefined. This is checked above the ADI, + * so there's no need to check it again here. */ - MPIU_Free(ptr); + /* FIXME: For shared memory windows, we should provide an implementation + * that uses a processor atomic operation. */ + if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || + (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { + mpi_errno = MPIDI_CH3I_Shm_fop_op(origin_addr, result_addr, datatype, + target_rank, target_disp, op, win_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + } + else { + MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); + MPIDI_RMA_Op_t *new_ptr = NULL; - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_FREE_MEM); + /* Append this operation to the RMA ops queue */ + mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + new_ptr->type = MPIDI_RMA_FETCH_AND_OP; + new_ptr->origin_addr = (void *) origin_addr; + new_ptr->origin_count = 1; + new_ptr->origin_datatype = datatype; + new_ptr->target_rank = target_rank; + new_ptr->target_disp = target_disp; + new_ptr->target_count = 1; + new_ptr->target_datatype = datatype; + new_ptr->result_addr = result_addr; + new_ptr->result_count = 1; + new_ptr->result_datatype = datatype; + new_ptr->op = op; + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_FETCH_AND_OP); return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ } diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c new file mode 100644 index 0000000..7cb36a5 --- /dev/null +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -0,0 +1,1756 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#include "mpidimpl.h" +#include "mpidrma.h" + + +/* ------------------------------------------------------------------------ */ +/* + * The following routines are the packet handlers for the packet types + * used above in the implementation of the RMA operations in terms + * of messages. + */ +/* ------------------------------------------------------------------------ */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Put +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_put_t *put_pkt = &pkt->put; + MPID_Request *req = NULL; + MPI_Aint type_size; + int complete = 0; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + MPID_Win *win_ptr; + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received put pkt"); + + if (put_pkt->count == 0) { + /* it's a 0-byte message sent just to decrement the + * completion counter. This happens only in + * post/start/complete/wait sync model; therefore, no need + * to check lock queue. */ + if (put_pkt->target_win_handle != MPI_WIN_NULL) { + MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr); + mpi_errno = + MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, put_pkt->flags, MPI_WIN_NULL); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + } + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + goto fn_exit; + } + + MPIU_Assert(put_pkt->target_win_handle != MPI_WIN_NULL); + MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, put_pkt->flags); + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); + + req->dev.user_buf = put_pkt->addr; + req->dev.user_count = put_pkt->count; + req->dev.target_win_handle = put_pkt->target_win_handle; + req->dev.source_win_handle = put_pkt->source_win_handle; + req->dev.flags = put_pkt->flags; + + if (MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)) { + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); + req->dev.datatype = put_pkt->datatype; + + MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); + req->dev.recv_data_sz = type_size * put_pkt->count; + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); + /* FIXME: Only change the handling of completion if + * post_data_receive reset the handler. There should + * be a cleaner way to do this */ + if (!req->dev.OnDataAvail) { + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete; + } + + /* return the number of bytes processed in this function */ + *buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len; + + if (complete) { + mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + if (complete) { + *rreqp = NULL; + goto fn_exit; + } + } + } + else { + /* derived datatype */ + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP_DERIVED_DT); + req->dev.datatype = MPI_DATATYPE_NULL; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete; + + req->dev.dtype_info = (MPIDI_RMA_dtype_info *) + MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); + if (!req->dev.dtype_info) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_RMA_dtype_info"); + } + + req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size); + if (!req->dev.dataloop) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + put_pkt->dataloop_size); + } + + /* if we received all of the dtype_info and dataloop, copy it + * now and call the handler, otherwise set the iov and let the + * channel copy it */ + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size) { + /* copy all of dtype_info and dataloop */ + MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); + MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), + put_pkt->dataloop_size); + + *buflen = + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size; + + /* All dtype data has been received, call req handler */ + mpi_errno = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete(vc, req, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); + if (complete) { + *rreqp = NULL; + goto fn_exit; + } + } + else { + req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) req->dev.dtype_info); + req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); + req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; + req->dev.iov[1].MPID_IOV_LEN = put_pkt->dataloop_size; + req->dev.iov_count = 2; + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete; + } + + } + + *rreqp = req; + + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); + } + + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT); + return mpi_errno; + fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Get +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_get_t *get_pkt = &pkt->get; + MPID_Request *req = NULL; + MPID_IOV iov[MPID_IOV_LIMIT]; + int complete; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + MPID_Win *win_ptr; + int mpi_errno = MPI_SUCCESS; + MPI_Aint type_size; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get pkt"); + + MPIU_Assert(get_pkt->target_win_handle != MPI_WIN_NULL); + MPID_Win_get_ptr(get_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, get_pkt->flags); + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + req = MPID_Request_create(); + req->dev.target_win_handle = get_pkt->target_win_handle; + req->dev.source_win_handle = get_pkt->source_win_handle; + req->dev.flags = get_pkt->flags; + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + if (MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)) { + /* basic datatype. send the data. */ + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp; + + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP); + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete; + req->kind = MPID_REQUEST_SEND; + + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + get_resp_pkt->request_handle = get_pkt->request_handle; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); + + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt->addr; + MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); + iov[1].MPID_IOV_LEN = get_pkt->count * type_size; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPIU_Object_set_ref(req, 0); + MPIDI_CH3_Request_destroy(req); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + /* --END ERROR HANDLING-- */ + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + } + else { + /* derived datatype. first get the dtype_info and dataloop. */ + + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP_DERIVED_DT); + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete; + req->dev.OnFinal = 0; + req->dev.user_buf = get_pkt->addr; + req->dev.user_count = get_pkt->count; + req->dev.datatype = MPI_DATATYPE_NULL; + req->dev.request_handle = get_pkt->request_handle; + + req->dev.dtype_info = (MPIDI_RMA_dtype_info *) + MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); + if (!req->dev.dtype_info) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_RMA_dtype_info"); + } + + req->dev.dataloop = MPIU_Malloc(get_pkt->dataloop_size); + if (!req->dev.dataloop) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + get_pkt->dataloop_size); + } + + /* if we received all of the dtype_info and dataloop, copy it + * now and call the handler, otherwise set the iov and let the + * channel copy it */ + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size) { + /* copy all of dtype_info and dataloop */ + MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); + MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), + get_pkt->dataloop_size); + + *buflen = + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size; + + /* All dtype data has been received, call req handler */ + mpi_errno = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete(vc, req, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET"); + if (complete) + *rreqp = NULL; + } + else { + req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; + req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); + req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; + req->dev.iov[1].MPID_IOV_LEN = get_pkt->dataloop_size; + req->dev.iov_count = 2; + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = req; + } + + } + fn_fail: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET); + return mpi_errno; + +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_accum_t *accum_pkt = &pkt->accum; + MPID_Request *req = NULL; + MPI_Aint true_lb, true_extent, extent; + void *tmp_buf = NULL; + int complete = 0; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + MPID_Win *win_ptr; + int mpi_errno = MPI_SUCCESS; + MPI_Aint type_size; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt"); + + MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); + MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags); + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); + *rreqp = req; + + req->dev.user_count = accum_pkt->count; + req->dev.op = accum_pkt->op; + req->dev.real_user_buf = accum_pkt->addr; + req->dev.target_win_handle = accum_pkt->target_win_handle; + req->dev.source_win_handle = accum_pkt->source_win_handle; + req->dev.flags = accum_pkt->flags; + + if (accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM) { + req->dev.resp_request_handle = accum_pkt->request_handle; + } + else { + req->dev.resp_request_handle = MPI_REQUEST_NULL; + } + + if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) { + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP); + req->dev.datatype = accum_pkt->datatype; + + MPIR_Type_get_true_extent_impl(accum_pkt->datatype, &true_lb, &true_extent); + MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent); + + /* Predefined types should always have zero lb */ + MPIU_Assert(true_lb == 0); + + tmp_buf = MPIU_Malloc(accum_pkt->count * (MPIR_MAX(extent, true_extent))); + if (!tmp_buf) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + accum_pkt->count * MPIR_MAX(extent, true_extent)); + } + + req->dev.user_buf = tmp_buf; + + MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size); + req->dev.recv_data_sz = type_size * accum_pkt->count; + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + /* FIXME: Only change the handling of completion if + * post_data_receive reset the handler. There should + * be a cleaner way to do this */ + if (!req->dev.OnDataAvail) { + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete; + } + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + if (complete) { + mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + if (complete) { + *rreqp = NULL; + goto fn_exit; + } + } + } + else { + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT); + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete; + req->dev.datatype = MPI_DATATYPE_NULL; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete; + + req->dev.dtype_info = (MPIDI_RMA_dtype_info *) + MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); + if (!req->dev.dtype_info) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_RMA_dtype_info"); + } + + req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size); + if (!req->dev.dataloop) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + accum_pkt->dataloop_size); + } + + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size) { + /* copy all of dtype_info and dataloop */ + MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); + MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), + accum_pkt->dataloop_size); + + *buflen = + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size; + + /* All dtype data has been received, call req handler */ + mpi_errno = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete(vc, req, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_ACCUMULATE"); + if (complete) { + *rreqp = NULL; + goto fn_exit; + } + } + else { + req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; + req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); + req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; + req->dev.iov[1].MPID_IOV_LEN = accum_pkt->dataloop_size; + req->dev.iov_count = 2; + *buflen = sizeof(MPIDI_CH3_Pkt_t); + } + + } + + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); + return mpi_errno; + fn_fail: + goto fn_exit; + +} + +/* Special accumulate for short data items entirely within the packet */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate_Immed +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_accum_immed_t *accum_pkt = &pkt->accum_immed; + MPID_Win *win_ptr; + MPI_Aint extent; + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate immedidate pkt"); + + MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); + MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags); + + /* return the number of bytes processed in this function */ + /* data_len == 0 (all within packet) */ + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent); + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + /* Data is already present */ + if (accum_pkt->op == MPI_REPLACE) { + /* no datatypes required */ + int len; + MPIU_Assign_trunc(len, (accum_pkt->count * extent), int); + /* FIXME: use immediate copy because this is short */ + MPIUI_Memcpy(accum_pkt->addr, accum_pkt->data, len); + } + else { + if (HANDLE_GET_KIND(accum_pkt->op) == HANDLE_KIND_BUILTIN) { + MPI_User_function *uop; + /* get the function by indexing into the op table */ + uop = MPIR_OP_HDL_TO_FN(accum_pkt->op); + (*uop) (accum_pkt->data, accum_pkt->addr, &(accum_pkt->count), &(accum_pkt->datatype)); + } + else { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OP, "**opnotpredefined", + "**opnotpredefined %d", accum_pkt->op); + } + } + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + + /* There are additional steps to take if this is a passive + * target RMA or the last operation from the source */ + + /* Here is the code executed in PutAccumRespComplete after the + * accumulation operation */ + MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); + + mpi_errno = MPIDI_CH3_Finish_rma_op_target(vc, win_ptr, TRUE, + accum_pkt->flags, accum_pkt->source_win_handle); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); + return mpi_errno; + fn_fail: + goto fn_exit; + +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_CAS +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &upkt.cas_resp; + MPIDI_CH3_Pkt_cas_t *cas_pkt = &pkt->cas; + MPID_Win *win_ptr; + MPID_Request *req; + MPI_Aint len; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt"); + + MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL); + MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, cas_pkt->flags); + + /* return the number of bytes processed in this function */ + /* data_len == 0 (all within packet) */ + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); + cas_resp_pkt->request_handle = cas_pkt->request_handle; + + /* Copy old value into the response packet */ + MPID_Datatype_get_size_macro(cas_pkt->datatype, len); + MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + + MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len); + + /* Compare and replace if equal */ + if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) { + MPIU_Memcpy(cas_pkt->addr, &cas_pkt->origin_data, len); + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + + /* Send the response packet */ + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + if (req != NULL) { + if (!MPID_Request_is_complete(req)) { + /* sending process is not completed, set proper OnDataAvail + (it is initialized to NULL by lower layer) */ + req->dev.target_win_handle = cas_pkt->target_win_handle; + req->dev.flags = cas_pkt->flags; + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetAccumRespComplete; + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + MPID_Request_release(req); + goto fn_exit; + } + else + MPID_Request_release(req); + } + + + /* There are additional steps to take if this is a passive + * target RMA or the last operation from the source */ + + mpi_errno = MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, cas_pkt->flags, MPI_WIN_NULL); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); + return mpi_errno; + fn_fail: + goto fn_exit; + +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_CASResp +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &pkt->cas_resp; + MPID_Request *req; + MPI_Aint len; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS response pkt"); + + MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); + MPID_Datatype_get_size_macro(req->dev.datatype, len); + + MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->data, len); + + MPIDI_CH3U_Request_complete(req); + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP); + return mpi_errno; + fn_fail: + goto fn_exit; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_FOP +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_fop_t *fop_pkt = &pkt->fop; + MPID_Request *req; + MPID_Win *win_ptr; + int data_complete = 0; + MPI_Aint len; + MPIU_CHKPMEM_DECL(1); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt"); + + MPIU_Assert(fop_pkt->target_win_handle != MPI_WIN_NULL); + MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, fop_pkt->flags); + + req = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + MPIU_Object_set_ref(req, 1); /* Ref is held by progress engine */ + *rreqp = NULL; + + req->dev.user_buf = NULL; /* will be set later */ + req->dev.user_count = 1; + req->dev.datatype = fop_pkt->datatype; + req->dev.op = fop_pkt->op; + req->dev.real_user_buf = fop_pkt->addr; + req->dev.target_win_handle = fop_pkt->target_win_handle; + req->dev.request_handle = fop_pkt->request_handle; + req->dev.flags = fop_pkt->flags; + + MPID_Datatype_get_size_macro(req->dev.datatype, len); + MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u)); + + /* Set up the user buffer and receive data if needed */ + if (len <= sizeof(fop_pkt->origin_data) || fop_pkt->op == MPI_NO_OP) { + req->dev.user_buf = fop_pkt->origin_data; + *buflen = sizeof(MPIDI_CH3_Pkt_t); + data_complete = 1; + } + else { + /* Data won't fit in the header, allocate temp space and receive it */ + MPIDI_msg_sz_t data_len; + void *data_buf; + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + req->dev.recv_data_sz = len; /* count == 1 for FOP */ + + MPIU_CHKPMEM_MALLOC(req->dev.user_buf, void *, len, mpi_errno, "**nomemreq"); + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &data_complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPComplete; + + if (!data_complete) { + *rreqp = req; + } + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + } + + if (data_complete) { + int fop_complete = 0; + mpi_errno = MPIDI_CH3_ReqHandler_FOPComplete(vc, req, &fop_complete); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + *rreqp = NULL; + } + + fn_exit: + MPIU_CHKPMEM_COMMIT(); + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_FOPResp +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &pkt->fop_resp; + MPID_Request *req; + int complete = 0; + MPI_Aint len; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP response pkt"); + + MPID_Request_get_ptr(fop_resp_pkt->request_handle, req); + MPID_Datatype_get_size_macro(req->dev.datatype, len); + + if (len <= sizeof(fop_resp_pkt->data)) { + MPIU_Memcpy(req->dev.user_buf, (void *) fop_resp_pkt->data, len); + *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; + } + else { + /* Data was too big to embed in the header */ + MPIDI_msg_sz_t data_len; + void *data_buf; + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + req->dev.recv_data_sz = len; /* count == 1 for FOP */ + *rreqp = req; + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, + "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_RESP"); + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + } + + if (complete) { + MPIDI_CH3U_Request_complete(req); + *rreqp = NULL; + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Get_AccumResp +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &pkt->get_accum_resp; + MPID_Request *req; + int complete; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + int mpi_errno = MPI_SUCCESS; + MPI_Aint type_size; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt"); + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req); + + MPID_Datatype_get_size_macro(req->dev.datatype, type_size); + req->dev.recv_data_sz = type_size * req->dev.user_count; + + *rreqp = req; + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_ACCUM_RESP"); + if (complete) { + MPIDI_CH3U_Request_complete(req); + *rreqp = NULL; + } + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); + return mpi_errno; + fn_fail: + goto fn_exit; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Lock +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt->lock; + MPID_Win *win_ptr = NULL; + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock pkt"); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + + MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr); + + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_pkt->lock_type) == 1) { + /* send lock granted packet. */ + mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, lock_pkt->source_win_handle); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + + else { + /* queue the lock information */ + MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; + + /* Note: This code is reached by the fechandadd rma tests */ + /* FIXME: MT: This may need to be done atomically. */ + + /* FIXME: Since we need to add to the tail of the list, + * we should maintain a tail pointer rather than traversing the + * list each time to find the tail. */ + curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; + prev_ptr = curr_ptr; + while (curr_ptr != NULL) { + prev_ptr = curr_ptr; + curr_ptr = curr_ptr->next; + } + + new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); + if (!new_ptr) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_Win_lock_queue"); + } + if (prev_ptr != NULL) + prev_ptr->next = new_ptr; + else + win_ptr->lock_queue = new_ptr; + + new_ptr->next = NULL; + new_ptr->lock_type = lock_pkt->lock_type; + new_ptr->source_win_handle = lock_pkt->source_win_handle; + new_ptr->vc = vc; + new_ptr->pt_single_op = NULL; + } + + *rreqp = NULL; + fn_fail: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_LockPutUnlock +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_lock_put_unlock_t *lock_put_unlock_pkt = &pkt->lock_put_unlock; + MPID_Win *win_ptr = NULL; + MPID_Request *req = NULL; + MPI_Aint type_size; + int complete; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_put_unlock pkt"); + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); + + req->dev.datatype = lock_put_unlock_pkt->datatype; + MPID_Datatype_get_size_macro(lock_put_unlock_pkt->datatype, type_size); + req->dev.recv_data_sz = type_size * lock_put_unlock_pkt->count; + req->dev.user_count = lock_put_unlock_pkt->count; + req->dev.target_win_handle = lock_put_unlock_pkt->target_win_handle; + + MPID_Win_get_ptr(lock_put_unlock_pkt->target_win_handle, win_ptr); + + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_put_unlock_pkt->lock_type) == 1) { + /* do the put. for this optimization, only basic datatypes supported. */ + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete; + req->dev.user_buf = lock_put_unlock_pkt->addr; + req->dev.source_win_handle = lock_put_unlock_pkt->source_win_handle; + req->dev.flags = lock_put_unlock_pkt->flags; + } + + else { + /* queue the information */ + MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; + + new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); + if (!new_ptr) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_Win_lock_queue"); + } + + new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op)); + if (new_ptr->pt_single_op == NULL) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_PT_single_op"); + } + + /* FIXME: MT: The queuing may need to be done atomically. */ + + curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; + prev_ptr = curr_ptr; + while (curr_ptr != NULL) { + prev_ptr = curr_ptr; + curr_ptr = curr_ptr->next; + } + + if (prev_ptr != NULL) + prev_ptr->next = new_ptr; + else + win_ptr->lock_queue = new_ptr; + + new_ptr->next = NULL; + new_ptr->lock_type = lock_put_unlock_pkt->lock_type; + new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle; + new_ptr->vc = vc; + + new_ptr->pt_single_op->type = MPIDI_RMA_PUT; + new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags; + new_ptr->pt_single_op->addr = lock_put_unlock_pkt->addr; + new_ptr->pt_single_op->count = lock_put_unlock_pkt->count; + new_ptr->pt_single_op->datatype = lock_put_unlock_pkt->datatype; + /* allocate memory to receive the data */ + new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz); + if (new_ptr->pt_single_op->data == NULL) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + req->dev.recv_data_sz); + } + + new_ptr->pt_single_op->data_recd = 0; + + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_PUT); + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete; + req->dev.user_buf = new_ptr->pt_single_op->data; + req->dev.lock_queue_entry = new_ptr; + } + + int (*fcn) (MPIDI_VC_t *, struct MPID_Request *, int *); + fcn = req->dev.OnDataAvail; + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, + "**ch3|postrecv", "**ch3|postrecv %s", + "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK"); + } + req->dev.OnDataAvail = fcn; + *rreqp = req; + + if (complete) { + mpi_errno = fcn(vc, req, &complete); + if (complete) { + *rreqp = NULL; + } + } + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, + "**ch3|postrecv", "**ch3|postrecv %s", + "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK"); + } + + fn_fail: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK); + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_LockGetUnlock +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_lock_get_unlock_t *lock_get_unlock_pkt = &pkt->lock_get_unlock; + MPID_Win *win_ptr = NULL; + MPI_Aint type_size; + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_get_unlock pkt"); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + + MPID_Win_get_ptr(lock_get_unlock_pkt->target_win_handle, win_ptr); + + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_get_unlock_pkt->lock_type) == 1) { + /* do the get. for this optimization, only basic datatypes supported. */ + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp; + MPID_Request *req; + MPID_IOV iov[MPID_IOV_LIMIT]; + + req = MPID_Request_create(); + req->dev.target_win_handle = lock_get_unlock_pkt->target_win_handle; + req->dev.source_win_handle = lock_get_unlock_pkt->source_win_handle; + req->dev.flags = lock_get_unlock_pkt->flags; + + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP); + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete; + req->kind = MPID_REQUEST_SEND; + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + get_resp_pkt->request_handle = lock_get_unlock_pkt->request_handle; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); + + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_get_unlock_pkt->addr; + MPID_Datatype_get_size_macro(lock_get_unlock_pkt->datatype, type_size); + iov[1].MPID_IOV_LEN = lock_get_unlock_pkt->count * type_size; + + mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPIU_Object_set_ref(req, 0); + MPIDI_CH3_Request_destroy(req); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + /* --END ERROR HANDLING-- */ + } + + else { + /* queue the information */ + MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; + + /* FIXME: MT: This may need to be done atomically. */ + + curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; + prev_ptr = curr_ptr; + while (curr_ptr != NULL) { + prev_ptr = curr_ptr; + curr_ptr = curr_ptr->next; + } + + new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); + if (!new_ptr) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_Win_lock_queue"); + } + new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op)); + if (new_ptr->pt_single_op == NULL) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_PT_Single_op"); + } + + if (prev_ptr != NULL) + prev_ptr->next = new_ptr; + else + win_ptr->lock_queue = new_ptr; + + new_ptr->next = NULL; + new_ptr->lock_type = lock_get_unlock_pkt->lock_type; + new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle; + new_ptr->vc = vc; + + new_ptr->pt_single_op->type = MPIDI_RMA_GET; + new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags; + new_ptr->pt_single_op->addr = lock_get_unlock_pkt->addr; + new_ptr->pt_single_op->count = lock_get_unlock_pkt->count; + new_ptr->pt_single_op->datatype = lock_get_unlock_pkt->datatype; + new_ptr->pt_single_op->data = NULL; + new_ptr->pt_single_op->request_handle = lock_get_unlock_pkt->request_handle; + new_ptr->pt_single_op->data_recd = 1; + } + + *rreqp = NULL; + + fn_fail: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK); + return mpi_errno; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_LockAccumUnlock +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_lock_accum_unlock_t *lock_accum_unlock_pkt = &pkt->lock_accum_unlock; + MPID_Request *req = NULL; + MPID_Win *win_ptr = NULL; + MPIDI_Win_lock_queue *curr_ptr = NULL, *prev_ptr = NULL, *new_ptr = NULL; + MPI_Aint type_size; + int complete; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_accum_unlock pkt"); + + /* no need to acquire the lock here because we need to receive the + * data into a temporary buffer first */ + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); + + req->dev.datatype = lock_accum_unlock_pkt->datatype; + MPID_Datatype_get_size_macro(lock_accum_unlock_pkt->datatype, type_size); + req->dev.recv_data_sz = type_size * lock_accum_unlock_pkt->count; + req->dev.user_count = lock_accum_unlock_pkt->count; + req->dev.target_win_handle = lock_accum_unlock_pkt->target_win_handle; + req->dev.flags = lock_accum_unlock_pkt->flags; + + /* queue the information */ + + new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); + if (!new_ptr) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_Win_lock_queue"); + } + + new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op)); + if (new_ptr->pt_single_op == NULL) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPIDI_PT_single_op"); + } + + MPID_Win_get_ptr(lock_accum_unlock_pkt->target_win_handle, win_ptr); + + /* FIXME: MT: The queuing may need to be done atomically. */ + + curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; + prev_ptr = curr_ptr; + while (curr_ptr != NULL) { + prev_ptr = curr_ptr; + curr_ptr = curr_ptr->next; + } + + if (prev_ptr != NULL) + prev_ptr->next = new_ptr; + else + win_ptr->lock_queue = new_ptr; + + new_ptr->next = NULL; + new_ptr->lock_type = lock_accum_unlock_pkt->lock_type; + new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle; + new_ptr->vc = vc; + + new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE; + new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags; + new_ptr->pt_single_op->addr = lock_accum_unlock_pkt->addr; + new_ptr->pt_single_op->count = lock_accum_unlock_pkt->count; + new_ptr->pt_single_op->datatype = lock_accum_unlock_pkt->datatype; + new_ptr->pt_single_op->op = lock_accum_unlock_pkt->op; + /* allocate memory to receive the data */ + new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz); + if (new_ptr->pt_single_op->data == NULL) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + req->dev.recv_data_sz); + } + + new_ptr->pt_single_op->data_recd = 0; + + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_ACCUM); + req->dev.user_buf = new_ptr->pt_single_op->data; + req->dev.lock_queue_entry = new_ptr; + + *rreqp = req; + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + /* FIXME: Only change the handling of completion if + * post_data_receive reset the handler. There should + * be a cleaner way to do this */ + if (!req->dev.OnDataAvail) { + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete; + } + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK"); + } + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + if (complete) { + mpi_errno = MPIDI_CH3_ReqHandler_SinglePutAccumComplete(vc, req, &complete); + if (complete) { + *rreqp = NULL; + } + } + fn_fail: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK); + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_GetResp +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &pkt->get_resp; + MPID_Request *req; + int complete; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + int mpi_errno = MPI_SUCCESS; + MPI_Aint type_size; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get response pkt"); + + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + MPID_Request_get_ptr(get_resp_pkt->request_handle, req); + + MPID_Datatype_get_size_macro(req->dev.datatype, type_size); + req->dev.recv_data_sz = type_size * req->dev.user_count; + + *rreqp = req; + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", + "MPIDI_CH3_PKT_GET_RESP"); + if (complete) { + MPIDI_CH3U_Request_complete(req); + *rreqp = NULL; + } + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP); + return mpi_errno; + fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_LockGranted +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &pkt->lock_granted; + MPID_Win *win_ptr = NULL; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock granted pkt"); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + + MPID_Win_get_ptr(lock_granted_pkt->source_win_handle, win_ptr); + /* set the remote_lock_state flag in the window */ + win_ptr->targets[lock_granted_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; + + *rreqp = NULL; + MPIDI_CH3_Progress_signal_completion(); + + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED); + return MPI_SUCCESS; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_PtRMADone +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_PtRMADone(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &pkt->pt_rma_done; + MPID_Win *win_ptr = NULL; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt"); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + + MPID_Win_get_ptr(pt_rma_done_pkt->source_win_handle, win_ptr); + MPIU_Assert(win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state != + MPIDI_CH3_WIN_LOCK_NONE); + + if (win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state == + MPIDI_CH3_WIN_LOCK_FLUSH) + win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = + MPIDI_CH3_WIN_LOCK_GRANTED; + else + win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; + + *rreqp = NULL; + MPIDI_CH3_Progress_signal_completion(); + + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); + return MPI_SUCCESS; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Unlock +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Unlock(MPIDI_VC_t * vc ATTRIBUTE((unused)), + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &pkt->unlock; + MPID_Win *win_ptr = NULL; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK); + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received unlock pkt"); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + MPID_Win_get_ptr(unlock_pkt->target_win_handle, win_ptr); + mpi_errno = MPIDI_CH3I_Release_lock(win_ptr); + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); + + MPIDI_CH3_Progress_signal_completion(); + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_Flush +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_Flush(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3_Pkt_flush_t *flush_pkt = &pkt->flush; + MPID_Win *win_ptr = NULL; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH); + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received flush pkt"); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + /* This is a flush request packet */ + if (flush_pkt->target_win_handle != MPI_WIN_NULL) { + MPID_Request *req = NULL; + + MPID_Win_get_ptr(flush_pkt->target_win_handle, win_ptr); + + flush_pkt->target_win_handle = MPI_WIN_NULL; + flush_pkt->target_rank = win_ptr->comm_ptr->rank; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); + + /* Release the request returned by iStartMsg */ + if (req != NULL) { + MPID_Request_release(req); + } + } + + /* This is a flush response packet */ + else { + MPID_Win_get_ptr(flush_pkt->source_win_handle, win_ptr); + win_ptr->targets[flush_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; + MPIDI_CH3_Progress_signal_completion(); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_Start_rma_op_target +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_Start_rma_op_target(MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET); + + /* Lock with NOCHECK is piggybacked on this message. We should be able to + * immediately grab the lock. Otherwise, there is a synchronization error. */ + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK && flags & MPIDI_CH3_PKT_FLAG_RMA_NOCHECK) { + int lock_acquired; + int lock_mode; + + if (flags & MPIDI_CH3_PKT_FLAG_RMA_SHARED) { + lock_mode = MPI_LOCK_SHARED; + } + else if (flags & MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE) { + lock_mode = MPI_LOCK_EXCLUSIVE; + } + else { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|rma_flags"); + } + + lock_acquired = MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_mode); + MPIU_ERR_CHKANDJUMP(!lock_acquired, mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|nocheck_invalid"); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_Finish_rma_op_target +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_rma_update, + MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET); + + /* This function should be called by the target process after each RMA + * operation is completed, to update synchronization state. */ + + /* Last RMA operation from source. If active target RMA, decrement window + * counter. */ + if (flags & MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE) { + MPIU_Assert(win_ptr->current_lock_type == MPID_LOCK_NONE); + + win_ptr->at_completion_counter -= 1; + MPIU_Assert(win_ptr->at_completion_counter >= 0); + + /* Signal the local process when the op counter reaches 0. */ + if (win_ptr->at_completion_counter == 0) + MPIDI_CH3_Progress_signal_completion(); + } + + /* If passive target RMA, release lock on window and grant next lock in the + * lock queue if there is any. If requested by the origin, send an ack back + * to indicate completion at the target. */ + else if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { + MPIU_Assert(win_ptr->current_lock_type != MPID_LOCK_NONE); + + if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) { + MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL); + mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, source_win_handle); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + } + + mpi_errno = MPIDI_CH3I_Release_lock(win_ptr); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + /* The local process may be waiting for the lock. Signal completion to + * wake it up, so it can attempt to grab the lock. */ + MPIDI_CH3_Progress_signal_completion(); + } + else if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) { + /* Ensure store instructions have been performed before flush call is + * finished on origin process. */ + OPA_read_write_barrier(); + + if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) { + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush; + MPID_Request *req = NULL; + + MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received piggybacked flush request"); + MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL); + + MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH); + flush_pkt->source_win_handle = source_win_handle; + flush_pkt->target_win_handle = MPI_WIN_NULL; + flush_pkt->target_rank = win_ptr->comm_ptr->rank; + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, + "**ch3|rma_msg"); + + /* Release the request returned by iStartMsg */ + if (req != NULL) { + MPID_Request_release(req); + } + } + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +/* ------------------------------------------------------------------------ */ +/* + * For debugging, we provide the following functions for printing the + * contents of an RMA packet + */ +/* ------------------------------------------------------------------------ */ +#ifdef MPICH_DBG_OUTPUT +int MPIDI_CH3_PktPrint_Put(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PUT\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->put.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->put.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->put.datatype)); + MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.dataloop_size)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->put.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->put.source_win_handle)); + /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->put.win_ptr)); */ + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_Get(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->get.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->get.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->get.datatype)); + MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.dataloop_size)); + MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request_handle)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->get.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->get.source_win_handle)); + /* + * MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request)); + * MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->get.win_ptr)); + */ + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_GetResp(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET_RESP\n")); + MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request_handle)); + /*MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request)); */ + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_Accumulate(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUMULATE\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum.datatype)); + MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.dataloop_size)); + MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum.op)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum.source_win_handle)); + /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */ + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_Accum_Immed(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUM_IMMED\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum_immed.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum_immed.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum_immed.datatype)); + MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum_immed.op)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum_immed.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum_immed.source_win_handle)); + /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */ + fflush(stdout); + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_Lock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK\n")); + MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock.lock_type)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock.source_win_handle)); + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_LockPutUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_PUT_UNLOCK\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_put_unlock.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_put_unlock.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_put_unlock.datatype)); + MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_put_unlock.lock_type)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_put_unlock.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_put_unlock.source_win_handle)); + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_LockAccumUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_accum_unlock.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_accum_unlock.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_accum_unlock.datatype)); + MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_accum_unlock.lock_type)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_accum_unlock.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle)); + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_LockGetUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GET_UNLOCK\n")); + MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_get_unlock.addr)); + MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_get_unlock.count)); + MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_get_unlock.datatype)); + MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_get_unlock.lock_type)); + MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_get_unlock.target_win_handle)); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_get_unlock.source_win_handle)); + MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->lock_get_unlock.request_handle)); + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_PtRMADone(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PT_RMA_DONE\n")); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle)); + return MPI_SUCCESS; +} + +int MPIDI_CH3_PktPrint_LockGranted(FILE * fp, MPIDI_CH3_Pkt_t * pkt) +{ + MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GRANTED\n")); + MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_granted.source_win_handle)); + return MPI_SUCCESS; +} +#endif diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 489309a..aeec089 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -15,16 +15,6 @@ categories: description : cvars that control behavior of ch3 cvars: - - name : MPIR_CVAR_CH3_RMA_ACC_IMMED - category : CH3 - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Use the immediate accumulate optimization - - name : MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD category : CH3 type : int @@ -354,100 +344,16 @@ void MPIDI_CH3_RMA_Init_Pvars(void) #define SYNC_POST_TAG 100 -static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr); -static int send_unlock_msg(int dest, MPID_Win * win_ptr); -/* static int send_flush_msg(int dest, MPID_Win *win_ptr); */ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank); -static int acquire_local_lock(MPID_Win * win_ptr, int lock_mode); -static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, - MPIDI_RMA_dtype_info * dtype_info, - void **dataloop, MPID_Request ** request); -static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, - MPIDI_RMA_dtype_info * dtype_info, - void **dataloop, MPID_Request ** request); -static int send_contig_acc_msg(MPIDI_RMA_Op_t *, MPID_Win *, - MPIDI_CH3_Pkt_flags_t flags, MPI_Win, MPI_Win, MPID_Request **); -static int send_immed_rmw_msg(MPIDI_RMA_Op_t *, MPID_Win *, - MPIDI_CH3_Pkt_flags_t flags, MPI_Win, MPI_Win, MPID_Request **); static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, int *wait_for_rma_done_pkt, MPIDI_CH3_Pkt_flags_t sync_flags); static int send_lock_put_or_acc(MPID_Win *, int); static int send_lock_get(MPID_Win *, int); -static inline int poke_progress_engine(void); static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list); static inline int rma_list_gc(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list, MPIDI_RMA_Op_t * last_elm, int *nDone); -static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, - const void *dataloop, MPI_Aint dataloop_sz, - const void *o_addr, int o_count, - MPI_Datatype o_datatype, MPID_Datatype ** combined_dtp); - -#define MPIDI_CH3I_ISSUE_RMA_OP(op_ptr_, win_ptr_, flags_, source_win_handle_, target_win_handle_,err_) \ - do { \ - switch ((op_ptr_)->type) \ - { \ - case (MPIDI_RMA_PUT): \ - case (MPIDI_RMA_ACCUMULATE): \ - (err_) = send_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \ - (target_win_handle_), &(op_ptr_)->dtype_info, \ - &(op_ptr_)->dataloop, &(op_ptr_)->request); \ - if (err_) { MPIU_ERR_POP(err_); } \ - break; \ - case (MPIDI_RMA_GET_ACCUMULATE): \ - if ((op_ptr_)->op == MPI_NO_OP) { \ - /* Note: Origin arguments are ignored for NO_OP, so we don't \ - * need to release a ref to the origin datatype. */ \ - \ - /* Convert the GAcc to a Get */ \ - (op_ptr_)->type = MPIDI_RMA_GET; \ - (op_ptr_)->origin_addr = (op_ptr_)->result_addr; \ - (op_ptr_)->origin_count = (op_ptr_)->result_count; \ - (op_ptr_)->origin_datatype = (op_ptr_)->result_datatype; \ - \ - (err_) = recv_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \ - (target_win_handle_), &(op_ptr_)->dtype_info, \ - &(op_ptr_)->dataloop, &(op_ptr_)->request); \ - } else { \ - (err_) = send_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \ - (target_win_handle_), &(op_ptr_)->dtype_info, \ - &(op_ptr_)->dataloop, &(op_ptr_)->request); \ - } \ - if (err_) { MPIU_ERR_POP(err_); } \ - break; \ - case MPIDI_RMA_ACC_CONTIG: \ - (err_) = send_contig_acc_msg((op_ptr_), (win_ptr_), (flags_), \ - (source_win_handle_), (target_win_handle_), \ - &(op_ptr_)->request); \ - if (err_) { MPIU_ERR_POP(err_); } \ - break; \ - case (MPIDI_RMA_GET): \ - (err_) = recv_rma_msg((op_ptr_), (win_ptr_), (flags_), \ - (source_win_handle_), (target_win_handle_), \ - &(op_ptr_)->dtype_info, \ - &(op_ptr_)->dataloop, &(op_ptr_)->request); \ - if (err_) { MPIU_ERR_POP(err_); } \ - break; \ - case (MPIDI_RMA_COMPARE_AND_SWAP): \ - case (MPIDI_RMA_FETCH_AND_OP): \ - (err_) = send_immed_rmw_msg((op_ptr_), (win_ptr_), (flags_), \ - (source_win_handle_), (target_win_handle_), \ - &(op_ptr_)->request); \ - if (err_) { MPIU_ERR_POP(err_); } \ - break; \ - \ - default: \ - MPIU_ERR_SETANDJUMP(err_,MPI_ERR_OTHER,"**winInvalidOp"); \ - } \ - } while (0) - #undef FUNCNAME #define FUNCNAME MPIDI_Win_fence @@ -584,8 +490,10 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) source_win_handle = win_ptr->handle; target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; - MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, - source_win_handle, target_win_handle, mpi_errno); + mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, + source_win_handle, target_win_handle); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); i++; curr_ops_cnt[curr_ptr->target_rank]++; @@ -697,790 +605,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) /* --END ERROR HANDLING-- */ } -/* create_datatype() creates a new struct datatype for the dtype_info - and the dataloop of the target datatype together with the user data */ -#undef FUNCNAME -#define FUNCNAME create_datatype -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, - const void *dataloop, MPI_Aint dataloop_sz, - const void *o_addr, int o_count, MPI_Datatype o_datatype, - MPID_Datatype ** combined_dtp) -{ - int mpi_errno = MPI_SUCCESS; - /* datatype_set_contents wants an array 'ints' which is the - * blocklens array with count prepended to it. So blocklens - * points to the 2nd element of ints to avoid having to copy - * blocklens into ints later. */ - int ints[4]; - int *blocklens = &ints[1]; - MPI_Aint displaces[3]; - MPI_Datatype datatypes[3]; - const int count = 3; - MPI_Datatype combined_datatype; - MPIDI_STATE_DECL(MPID_STATE_CREATE_DATATYPE); - - MPIDI_FUNC_ENTER(MPID_STATE_CREATE_DATATYPE); - - /* create datatype */ - displaces[0] = MPIU_PtrToAint(dtype_info); - blocklens[0] = sizeof(*dtype_info); - datatypes[0] = MPI_BYTE; - - displaces[1] = MPIU_PtrToAint(dataloop); - MPIU_Assign_trunc(blocklens[1], dataloop_sz, int); - datatypes[1] = MPI_BYTE; - - displaces[2] = MPIU_PtrToAint(o_addr); - blocklens[2] = o_count; - datatypes[2] = o_datatype; - - mpi_errno = MPID_Type_struct(count, blocklens, displaces, datatypes, &combined_datatype); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - ints[0] = count; - - MPID_Datatype_get_ptr(combined_datatype, *combined_dtp); - mpi_errno = MPID_Datatype_set_contents(*combined_dtp, MPI_COMBINER_STRUCT, count + 1, /* ints (cnt,blklen) */ - count, /* aints (disps) */ - count, /* types */ - ints, displaces, datatypes); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - /* Commit datatype */ - - MPID_Dataloop_create(combined_datatype, - &(*combined_dtp)->dataloop, - &(*combined_dtp)->dataloop_size, - &(*combined_dtp)->dataloop_depth, MPID_DATALOOP_HOMOGENEOUS); - - /* create heterogeneous dataloop */ - MPID_Dataloop_create(combined_datatype, - &(*combined_dtp)->hetero_dloop, - &(*combined_dtp)->hetero_dloop_size, - &(*combined_dtp)->hetero_dloop_depth, MPID_DATALOOP_HETEROGENEOUS); - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_CREATE_DATATYPE); - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME -#define FUNCNAME send_rma_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, - MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request) -{ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_put_t *put_pkt = &upkt.put; - MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum; - MPID_IOV iov[MPID_IOV_LIMIT]; - int mpi_errno = MPI_SUCCESS; - int origin_dt_derived, target_dt_derived, iovcnt; - MPI_Aint origin_type_size; - MPIDI_VC_t *vc; - MPID_Comm *comm_ptr; - MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL; - MPID_Request *resp_req = NULL; - MPIU_CHKPMEM_DECL(1); - MPIDI_STATE_DECL(MPID_STATE_SEND_RMA_MSG); - MPIDI_STATE_DECL(MPID_STATE_MEMCPY); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_RMA_MSG); - - *request = NULL; - - if (rma_op->type == MPIDI_RMA_PUT) { - MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); - put_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - put_pkt->flags = flags; - put_pkt->count = rma_op->target_count; - put_pkt->datatype = rma_op->target_datatype; - put_pkt->dataloop_size = 0; - put_pkt->target_win_handle = target_win_handle; - put_pkt->source_win_handle = source_win_handle; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt; - iov[0].MPID_IOV_LEN = sizeof(*put_pkt); - } - else if (rma_op->type == MPIDI_RMA_GET_ACCUMULATE) { - /* Create a request for the GACC response. Store the response buf, count, and - * datatype in it, and pass the request's handle in the GACC packet. When the - * response comes from the target, it will contain the request handle. */ - resp_req = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - - MPIU_Object_set_ref(resp_req, 2); - - resp_req->dev.user_buf = rma_op->result_addr; - resp_req->dev.user_count = rma_op->result_count; - resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = target_win_handle; - resp_req->dev.source_win_handle = source_win_handle; - - if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { - MPID_Datatype *result_dtp = NULL; - MPID_Datatype_get_ptr(resp_req->dev.datatype, result_dtp); - resp_req->dev.datatype_ptr = result_dtp; - /* this will cause the datatype to be freed when the - * request is freed. */ - } - - /* Note: Get_accumulate uses the same packet type as accumulate */ - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); - accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - accum_pkt->flags = flags; - accum_pkt->count = rma_op->target_count; - accum_pkt->datatype = rma_op->target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = rma_op->op; - accum_pkt->target_win_handle = target_win_handle; - accum_pkt->source_win_handle = source_win_handle; - accum_pkt->request_handle = resp_req->handle; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; - iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); - } - else { - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); - accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - accum_pkt->flags = flags; - accum_pkt->count = rma_op->target_count; - accum_pkt->datatype = rma_op->target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = rma_op->op; - accum_pkt->target_win_handle = target_win_handle; - accum_pkt->source_win_handle = source_win_handle; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; - iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); - } - - /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, - * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); - * fflush(stdout); - */ - - comm_ptr = win_ptr->comm_ptr; - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - - if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) { - origin_dt_derived = 1; - MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp); - } - else { - origin_dt_derived = 0; - } - - if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) { - target_dt_derived = 1; - MPID_Datatype_get_ptr(rma_op->target_datatype, target_dtp); - } - else { - target_dt_derived = 0; - } - - if (target_dt_derived) { - /* derived datatype on target. fill derived datatype info */ - dtype_info->is_contig = target_dtp->is_contig; - dtype_info->max_contig_blocks = target_dtp->max_contig_blocks; - dtype_info->size = target_dtp->size; - dtype_info->extent = target_dtp->extent; - dtype_info->dataloop_size = target_dtp->dataloop_size; - dtype_info->dataloop_depth = target_dtp->dataloop_depth; - dtype_info->eltype = target_dtp->eltype; - dtype_info->dataloop = target_dtp->dataloop; - dtype_info->ub = target_dtp->ub; - dtype_info->lb = target_dtp->lb; - dtype_info->true_ub = target_dtp->true_ub; - dtype_info->true_lb = target_dtp->true_lb; - dtype_info->has_sticky_ub = target_dtp->has_sticky_ub; - dtype_info->has_sticky_lb = target_dtp->has_sticky_lb; - - MPIU_CHKPMEM_MALLOC(*dataloop, void *, target_dtp->dataloop_size, mpi_errno, "dataloop"); - - MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY); - MPIU_Memcpy(*dataloop, target_dtp->dataloop, target_dtp->dataloop_size); - MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY); - /* the dataloop can have undefined padding sections, so we need to let - * valgrind know that it is OK to pass this data to writev later on */ - MPL_VG_MAKE_MEM_DEFINED(*dataloop, target_dtp->dataloop_size); - - if (rma_op->type == MPIDI_RMA_PUT) { - put_pkt->dataloop_size = target_dtp->dataloop_size; - } - else { - accum_pkt->dataloop_size = target_dtp->dataloop_size; - } - } - - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - - if (!target_dt_derived) { - /* basic datatype on target */ - if (!origin_dt_derived) { - /* basic datatype on origin */ - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr; - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; - iovcnt = 2; - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - else { - /* derived datatype on origin */ - *request = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(*request == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - - MPIU_Object_set_ref(*request, 2); - (*request)->kind = MPID_REQUEST_SEND; - - (*request)->dev.segment_ptr = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, - "**nomem", "**nomem %s", "MPID_Segment_alloc"); - - (*request)->dev.datatype_ptr = origin_dtp; - /* this will cause the datatype to be freed when the request - * is freed. */ - MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count, - rma_op->origin_datatype, (*request)->dev.segment_ptr, 0); - (*request)->dev.segment_first = 0; - (*request)->dev.segment_size = rma_op->origin_count * origin_type_size; - - (*request)->dev.OnFinal = 0; - (*request)->dev.OnDataAvail = 0; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = - vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - } - else { - /* derived datatype on target */ - MPID_Datatype *combined_dtp = NULL; - - *request = MPID_Request_create(); - if (*request == NULL) { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - } - - MPIU_Object_set_ref(*request, 2); - (*request)->kind = MPID_REQUEST_SEND; - - (*request)->dev.segment_ptr = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((*request)->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, - "**nomem", "**nomem %s", "MPID_Segment_alloc"); - - /* create a new datatype containing the dtype_info, dataloop, and origin data */ - - mpi_errno = - create_datatype(dtype_info, *dataloop, target_dtp->dataloop_size, rma_op->origin_addr, - rma_op->origin_count, rma_op->origin_datatype, &combined_dtp); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - (*request)->dev.datatype_ptr = combined_dtp; - /* combined_datatype will be freed when request is freed */ - - MPID_Segment_init(MPI_BOTTOM, 1, combined_dtp->handle, (*request)->dev.segment_ptr, 0); - (*request)->dev.segment_first = 0; - (*request)->dev.segment_size = combined_dtp->size; - - (*request)->dev.OnFinal = 0; - (*request)->dev.OnDataAvail = 0; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = vc->sendNoncontig_fn(vc, *request, iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - - /* we're done with the datatypes */ - if (origin_dt_derived) - MPID_Datatype_release(origin_dtp); - MPID_Datatype_release(target_dtp); - } - - /* This operation can generate two requests; one for inbound and one for - * outbound data. */ - if (resp_req != NULL) { - if (*request != NULL) { - /* If we have both inbound and outbound requests (i.e. GACC - * operation), we need to ensure that the source buffer is - * available and that the response data has been received before - * informing the origin that this operation is complete. Because - * the update needs to be done atomically at the target, they will - * not send back data until it has been received. Therefore, - * completion of the response request implies that the send request - * has completed. - * - * Therefore: refs on the response request are set to two: one is - * held by the progress engine and the other by the RMA op - * completion code. Refs on the outbound request are set to one; - * it will be completed by the progress engine. - */ - - MPID_Request_release(*request); - *request = resp_req; - - } - else { - *request = resp_req; - } - - /* For error checking */ - resp_req = NULL; - } - - fn_exit: - MPIU_CHKPMEM_COMMIT(); - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_RMA_MSG); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - if (resp_req) { - MPID_Request_release(resp_req); - } - if (*request) { - MPIU_CHKPMEM_REAP(); - if ((*request)->dev.datatype_ptr) - MPID_Datatype_release((*request)->dev.datatype_ptr); - MPID_Request_release(*request); - } - *request = NULL; - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - -/* - * Use this for contiguous accumulate operations - */ -#undef FUNCNAME -#define FUNCNAME send_contig_acc_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_contig_acc_msg(MPIDI_RMA_Op_t * rma_op, - MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, MPID_Request ** request) -{ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_accum_t *accum_pkt = &upkt.accum; - MPID_IOV iov[MPID_IOV_LIMIT]; - int mpi_errno = MPI_SUCCESS; - int iovcnt; - MPI_Aint origin_type_size; - MPIDI_VC_t *vc; - MPID_Comm *comm_ptr; - size_t len; - MPIDI_STATE_DECL(MPID_STATE_SEND_CONTIG_ACC_MSG); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_CONTIG_ACC_MSG); - - *request = NULL; - - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - /* FIXME: Make this size check efficient and match the packet type */ - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS * sizeof(int)) { - MPIDI_CH3_Pkt_accum_immed_t *accumi_pkt = &upkt.accum_immed; - void *dest = accumi_pkt->data, *src = rma_op->origin_addr; - - MPIDI_Pkt_init(accumi_pkt, MPIDI_CH3_PKT_ACCUM_IMMED); - accumi_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - accumi_pkt->flags = flags; - accumi_pkt->count = rma_op->target_count; - accumi_pkt->datatype = rma_op->target_datatype; - accumi_pkt->op = rma_op->op; - accumi_pkt->target_win_handle = target_win_handle; - accumi_pkt->source_win_handle = source_win_handle; - - switch (len) { - case 1: - *(uint8_t *) dest = *(uint8_t *) src; - break; - case 2: - *(uint16_t *) dest = *(uint16_t *) src; - break; - case 4: - *(uint32_t *) dest = *(uint32_t *) src; - break; - case 8: - *(uint64_t *) dest = *(uint64_t *) src; - break; - default: - MPIU_Memcpy(accumi_pkt->data, (void *) rma_op->origin_addr, len); - } - comm_ptr = win_ptr->comm_ptr; - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, accumi_pkt, sizeof(*accumi_pkt), request); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - goto fn_exit; - } - - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); - accum_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - accum_pkt->flags = flags; - accum_pkt->count = rma_op->target_count; - accum_pkt->datatype = rma_op->target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = rma_op->op; - accum_pkt->target_win_handle = target_win_handle; - accum_pkt->source_win_handle = source_win_handle; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; - iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); - - /* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, - * rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); - * fflush(stdout); - */ - - comm_ptr = win_ptr->comm_ptr; - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - - - /* basic datatype on target */ - /* basic datatype on origin */ - /* FIXME: This is still very heavyweight for a small message operation, - * such as a single word update */ - /* One possibility is to use iStartMsg with a buffer that is just large - * enough, though note that nemesis has an optimization for this */ - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr; - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; - iovcnt = 2; - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, request); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_CONTIG_ACC_MSG); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - if (*request) { - MPID_Request_release(*request); - } - *request = NULL; - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -/* - * Initiate an immediate RMW accumulate operation - */ -#undef FUNCNAME -#define FUNCNAME send_immed_rmw_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_immed_rmw_msg(MPIDI_RMA_Op_t * rma_op, - MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, MPID_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Request *rmw_req = NULL, *resp_req = NULL; - MPIDI_VC_t *vc; - MPID_Comm *comm_ptr; - MPI_Aint len; - MPIDI_STATE_DECL(MPID_STATE_SEND_IMMED_RMW_MSG); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_IMMED_RMW_MSG); - - *request = NULL; - - /* Create a request for the RMW response. Store the origin buf, count, and - * datatype in it, and pass the request's handle RMW packet. When the - * response comes from the target, it will contain the request handle. */ - resp_req = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - *request = resp_req; - - /* Set refs on the request to 2: one for the response message, and one for - * the partial completion handler */ - MPIU_Object_set_ref(resp_req, 2); - - resp_req->dev.user_buf = rma_op->result_addr; - resp_req->dev.user_count = rma_op->result_count; - resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = target_win_handle; - resp_req->dev.source_win_handle = source_win_handle; - - /* REQUIRE: All datatype arguments must be of the same, builtin - * type and counts must be 1. */ - MPID_Datatype_get_size_macro(rma_op->origin_datatype, len); - comm_ptr = win_ptr->comm_ptr; - - if (rma_op->type == MPIDI_RMA_COMPARE_AND_SWAP) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_cas_t *cas_pkt = &upkt.cas; - - MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); - - MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); - - cas_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - cas_pkt->flags = flags; - cas_pkt->datatype = rma_op->target_datatype; - cas_pkt->target_win_handle = target_win_handle; - cas_pkt->request_handle = resp_req->handle; - - MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); - MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); - - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_pkt, sizeof(*cas_pkt), &rmw_req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - - if (rmw_req != NULL) { - MPID_Request_release(rmw_req); - } - } - - else if (rma_op->type == MPIDI_RMA_FETCH_AND_OP) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_fop_t *fop_pkt = &upkt.fop; - - MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u)); - - MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); - - fop_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - fop_pkt->flags = flags; - fop_pkt->datatype = rma_op->target_datatype; - fop_pkt->target_win_handle = target_win_handle; - fop_pkt->request_handle = resp_req->handle; - fop_pkt->op = rma_op->op; - - if (len <= sizeof(fop_pkt->origin_data) || rma_op->op == MPI_NO_OP) { - /* Embed FOP data in the packet header */ - if (rma_op->op != MPI_NO_OP) { - MPIU_Memcpy(fop_pkt->origin_data, rma_op->origin_addr, len); - } - - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_pkt, sizeof(*fop_pkt), &rmw_req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - - if (rmw_req != NULL) { - MPID_Request_release(rmw_req); - } - } - else { - /* Data is too big to copy into the FOP header, use an IOV to send it */ - MPID_IOV iov[MPID_IOV_LIMIT]; - - rmw_req = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(rmw_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - MPIU_Object_set_ref(rmw_req, 1); - - rmw_req->dev.OnFinal = 0; - rmw_req->dev.OnDataAvail = 0; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_pkt; - iov[0].MPID_IOV_LEN = sizeof(*fop_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) rma_op->origin_addr; - iov[1].MPID_IOV_LEN = len; /* count == 1 */ - - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iSendv(vc, rmw_req, iov, 2); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - } - else { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_IMMED_RMW_MSG); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - if (*request) { - MPID_Request_release(*request); - } - *request = NULL; - if (rmw_req) { - MPID_Request_release(rmw_req); - } - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - - -#undef FUNCNAME -#define FUNCNAME recv_rma_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int recv_rma_msg(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, - MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle, - MPI_Win target_win_handle, - MPIDI_RMA_dtype_info * dtype_info, void **dataloop, MPID_Request ** request) -{ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_get_t *get_pkt = &upkt.get; - int mpi_errno = MPI_SUCCESS; - MPIDI_VC_t *vc; - MPID_Comm *comm_ptr; - MPID_Request *req = NULL; - MPID_Datatype *dtp; - MPID_IOV iov[MPID_IOV_LIMIT]; - MPIU_CHKPMEM_DECL(1); - MPIDI_STATE_DECL(MPID_STATE_RECV_RMA_MSG); - MPIDI_STATE_DECL(MPID_STATE_MEMCPY); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_RECV_RMA_MSG); - - /* create a request, store the origin buf, cnt, datatype in it, - * and pass a handle to it in the get packet. When the get - * response comes from the target, it will contain the request - * handle. */ - req = MPID_Request_create(); - if (req == NULL) { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - } - - *request = req; - - MPIU_Object_set_ref(req, 2); - - req->dev.user_buf = rma_op->origin_addr; - req->dev.user_count = rma_op->origin_count; - req->dev.datatype = rma_op->origin_datatype; - req->dev.target_win_handle = MPI_WIN_NULL; - req->dev.source_win_handle = source_win_handle; - if (!MPIR_DATATYPE_IS_PREDEFINED(req->dev.datatype)) { - MPID_Datatype_get_ptr(req->dev.datatype, dtp); - req->dev.datatype_ptr = dtp; - /* this will cause the datatype to be freed when the - * request is freed. */ - } - - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->base_addrs[rma_op->target_rank] + - win_ptr->disp_units[rma_op->target_rank] * rma_op->target_disp; - get_pkt->flags = flags; - get_pkt->count = rma_op->target_count; - get_pkt->datatype = rma_op->target_datatype; - get_pkt->request_handle = req->handle; - get_pkt->target_win_handle = target_win_handle; - get_pkt->source_win_handle = source_win_handle; - -/* printf("send pkt: type %d, addr %d, count %d, base %d\n", rma_pkt->type, - rma_pkt->addr, rma_pkt->count, win_ptr->base_addrs[rma_op->target_rank]); - fflush(stdout); -*/ - - comm_ptr = win_ptr->comm_ptr; - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - - if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->target_datatype)) { - /* basic datatype on target. simply send the get_pkt. */ - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, get_pkt, sizeof(*get_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - } - else { - /* derived datatype on target. fill derived datatype info and - * send it along with get_pkt. */ - - MPID_Datatype_get_ptr(rma_op->target_datatype, dtp); - dtype_info->is_contig = dtp->is_contig; - dtype_info->max_contig_blocks = dtp->max_contig_blocks; - dtype_info->size = dtp->size; - dtype_info->extent = dtp->extent; - dtype_info->dataloop_size = dtp->dataloop_size; - dtype_info->dataloop_depth = dtp->dataloop_depth; - dtype_info->eltype = dtp->eltype; - dtype_info->dataloop = dtp->dataloop; - dtype_info->ub = dtp->ub; - dtype_info->lb = dtp->lb; - dtype_info->true_ub = dtp->true_ub; - dtype_info->true_lb = dtp->true_lb; - dtype_info->has_sticky_ub = dtp->has_sticky_ub; - dtype_info->has_sticky_lb = dtp->has_sticky_lb; - - MPIU_CHKPMEM_MALLOC(*dataloop, void *, dtp->dataloop_size, mpi_errno, "dataloop"); - - MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY); - MPIU_Memcpy(*dataloop, dtp->dataloop, dtp->dataloop_size); - MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY); - - /* the dataloop can have undefined padding sections, so we need to let - * valgrind know that it is OK to pass this data to writev later on */ - MPL_VG_MAKE_MEM_DEFINED(*dataloop, dtp->dataloop_size); - - get_pkt->dataloop_size = dtp->dataloop_size; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt; - iov[0].MPID_IOV_LEN = sizeof(*get_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) dtype_info; - iov[1].MPID_IOV_LEN = sizeof(*dtype_info); - iov[2].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) * dataloop; - iov[2].MPID_IOV_LEN = dtp->dataloop_size; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, 3, &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - - /* release the target datatype */ - MPID_Datatype_release(dtp); - } - - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - - /* release the request returned by iStartMsg or iStartMsgv */ - if (req != NULL) { - MPID_Request_release(req); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_RECV_RMA_MSG); - return mpi_errno; - - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - MPIU_CHKPMEM_REAP(); - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - #undef FUNCNAME #define FUNCNAME MPIDI_Win_post @@ -1903,8 +1027,10 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) source_win_handle = win_ptr->handle; target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; - MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, - source_win_handle, target_win_handle, mpi_errno); + mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, + source_win_handle, target_win_handle); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); i++; curr_ops_cnt[curr_ptr->target_rank]++; @@ -1992,9 +1118,9 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) /* free the group stored in window */ MPIR_Group_release(win_ptr->start_group_ptr); - win_ptr->start_group_ptr = NULL; - - fn_exit: + win_ptr->start_group_ptr = NULL; + + fn_exit: MPIU_CHKLMEM_FREEALL(); MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_COMPLETE); return mpi_errno; @@ -2060,6 +1186,7 @@ int MPIDI_Win_wait(MPID_Win * win_ptr) /* --END ERROR HANDLING-- */ } + #undef FUNCNAME #define FUNCNAME MPIDI_Win_test #undef FCNAME @@ -2106,8 +1233,6 @@ int MPIDI_Win_test(MPID_Win * win_ptr, int *flag) } - - #undef FUNCNAME #define FUNCNAME MPIDI_Win_lock #undef FCNAME @@ -2779,33 +1904,6 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr) #undef FUNCNAME -#define FUNCNAME MPIDI_Win_sync -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_Win_sync(MPID_Win * win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SYNC); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SYNC); - - MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK && - win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL, - mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - - OPA_read_write_barrier(); - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SYNC); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME #define FUNCNAME do_passive_target_rma #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) @@ -2934,8 +2032,10 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, source_win_handle = win_ptr->handle; } - MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, source_win_handle, - target_win_handle, mpi_errno); + mpi_errno = MPIDI_CH3I_Issue_rma_op(curr_ptr, win_ptr, flags, source_win_handle, + target_win_handle); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); /* If the request is null, we can remove it immediately */ if (!curr_ptr->request) { @@ -2996,85 +2096,25 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, /* --END ERROR HANDLING-- */ } - #undef FUNCNAME -#define FUNCNAME send_lock_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_lock_t *lock_pkt = &upkt.lock; - MPID_Request *req = NULL; - MPIDI_VC_t *vc; - MPIDI_STATE_DECL(MPID_STATE_SEND_LOCK_MSG); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_LOCK_MSG); - - MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED); - - MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); - - MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); - lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; - lock_pkt->source_win_handle = win_ptr->handle; - lock_pkt->lock_type = lock_type; - - win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED; - win_ptr->targets[dest].remote_lock_mode = lock_type; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_pkt, sizeof(*lock_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - /* release the request returned by iStartMsg */ - if (req != NULL) { - MPID_Request_release(req); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_LOCK_MSG); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME -#define FUNCNAME acquire_local_lock +#define FUNCNAME MPIDI_Win_sync #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int acquire_local_lock(MPID_Win * win_ptr, int lock_type) +int MPIDI_Win_sync(MPID_Win * win_ptr) { int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_ACQUIRE_LOCAL_LOCK); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SYNC); - /* poke the progress engine until the local lock is granted */ - if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { - MPID_Progress_state progress_state; + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SYNC); - MPID_Progress_start(&progress_state); - while (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { - mpi_errno = MPID_Progress_wait(&progress_state); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPID_Progress_end(&progress_state); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); - } - /* --END ERROR HANDLING-- */ - } - MPID_Progress_end(&progress_state); - } + MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK && + win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL, + mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; - win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_mode = lock_type; + OPA_read_write_barrier(); fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_ACQUIRE_LOCAL_LOCK); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SYNC); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: @@ -3129,99 +2169,6 @@ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank) #undef FUNCNAME -#define FUNCNAME send_unlock_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_unlock_msg(int dest, MPID_Win * win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &upkt.unlock; - MPID_Request *req = NULL; - MPIDI_VC_t *vc; - MPIDI_STATE_DECL(MPID_STATE_SEND_UNLOCK_MSG); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_UNLOCK_MSG); - - MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_GRANTED); - - MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); - - /* Send a lock packet over to the target. wait for the lock_granted - * reply. Then do all the RMA ops. */ - - MPIDI_Pkt_init(unlock_pkt, MPIDI_CH3_PKT_UNLOCK); - unlock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; - - /* Reset the local state of the target to unlocked */ - win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, unlock_pkt, sizeof(*unlock_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - /* Release the request returned by iStartMsg */ - if (req != NULL) { - MPID_Request_release(req); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_UNLOCK_MSG); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -/* Commented out function to squash a warning, but retaining the code - * for later use. */ -#if 0 -#undef FUNCNAME -#define FUNCNAME send_flush_msg -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int send_flush_msg(int dest, MPID_Win * win_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush; - MPID_Request *req = NULL; - MPIDI_VC_t *vc; - MPIDI_STATE_DECL(MPID_STATE_SEND_FLUSH_MSG); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SEND_FLUSH_MSG); - - MPIU_Assert(win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_FLUSH); - - MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, dest, &vc); - - MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH); - flush_pkt->target_win_handle = win_ptr->all_win_handles[dest]; - flush_pkt->source_win_handle = win_ptr->handle; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - /* Release the request returned by iStartMsg */ - if (req != NULL) { - MPID_Request_release(req); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SEND_FLUSH_MSG); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} -#endif - - -#undef FUNCNAME #define FUNCNAME send_lock_put_or_acc #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) @@ -3523,1549 +2470,8 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank) } /* ------------------------------------------------------------------------ */ -/* - * Utility routines - */ -/* ------------------------------------------------------------------------ */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Send_lock_granted_pkt -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3I_Send_lock_granted_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPI_Win source_win_handle) -{ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &upkt.lock_granted; - MPID_Request *req = NULL; - int mpi_errno; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT); - - /* send lock granted packet */ - MPIDI_Pkt_init(lock_granted_pkt, MPIDI_CH3_PKT_LOCK_GRANTED); - lock_granted_pkt->source_win_handle = source_win_handle; - lock_granted_pkt->target_rank = win_ptr->comm_ptr->rank; - - MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, - (MPIU_DBG_FDEST, "sending lock granted pkt on vc=%p, source_win_handle=%#08x", - vc, lock_granted_pkt->source_win_handle)); - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_granted_pkt, sizeof(*lock_granted_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - if (mpi_errno) { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - - if (req != NULL) { - MPID_Request_release(req); - } - - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_LOCK_GRANTED_PKT); - - return mpi_errno; -} - -/* ------------------------------------------------------------------------ */ -/* - * The following routines are the packet handlers for the packet types - * used above in the implementation of the RMA operations in terms - * of messages. - */ -/* ------------------------------------------------------------------------ */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Put -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_put_t *put_pkt = &pkt->put; - MPID_Request *req = NULL; - MPI_Aint type_size; - int complete = 0; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - MPID_Win *win_ptr; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received put pkt"); - - if (put_pkt->count == 0) { - /* it's a 0-byte message sent just to decrement the - * completion counter. This happens only in - * post/start/complete/wait sync model; therefore, no need - * to check lock queue. */ - if (put_pkt->target_win_handle != MPI_WIN_NULL) { - MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr); - mpi_errno = - MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, put_pkt->flags, MPI_WIN_NULL); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - goto fn_exit; - } - - MPIU_Assert(put_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, put_pkt->flags); - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); - - req->dev.user_buf = put_pkt->addr; - req->dev.user_count = put_pkt->count; - req->dev.target_win_handle = put_pkt->target_win_handle; - req->dev.source_win_handle = put_pkt->source_win_handle; - req->dev.flags = put_pkt->flags; - - if (MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)) { - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); - req->dev.datatype = put_pkt->datatype; - - MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); - req->dev.recv_data_sz = type_size * put_pkt->count; - - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); - /* FIXME: Only change the handling of completion if - * post_data_receive reset the handler. There should - * be a cleaner way to do this */ - if (!req->dev.OnDataAvail) { - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete; - } - - /* return the number of bytes processed in this function */ - *buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len; - - if (complete) { - mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } - } - else { - /* derived datatype */ - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP_DERIVED_DT); - req->dev.datatype = MPI_DATATYPE_NULL; - req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete; - - req->dev.dtype_info = (MPIDI_RMA_dtype_info *) - MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); - if (!req->dev.dtype_info) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_RMA_dtype_info"); - } - - req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size); - if (!req->dev.dataloop) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - put_pkt->dataloop_size); - } - - /* if we received all of the dtype_info and dataloop, copy it - * now and call the handler, otherwise set the iov and let the - * channel copy it */ - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size) { - /* copy all of dtype_info and dataloop */ - MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); - MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - put_pkt->dataloop_size); - - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size; - - /* All dtype data has been received, call req handler */ - mpi_errno = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete(vc, req, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } - else { - req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) req->dev.dtype_info); - req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); - req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = put_pkt->dataloop_size; - req->dev.iov_count = 2; - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete; - } - - } - - *rreqp = req; - - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); - } - - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Get -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_get_t *get_pkt = &pkt->get; - MPID_Request *req = NULL; - MPID_IOV iov[MPID_IOV_LIMIT]; - int complete; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - MPID_Win *win_ptr; - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get pkt"); - - MPIU_Assert(get_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(get_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, get_pkt->flags); - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - req = MPID_Request_create(); - req->dev.target_win_handle = get_pkt->target_win_handle; - req->dev.source_win_handle = get_pkt->source_win_handle; - req->dev.flags = get_pkt->flags; - - /* here we increment the Active Target counter to guarantee the GET-like - operation are completed when counter reaches zero. */ - win_ptr->at_completion_counter++; - - if (MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)) { - /* basic datatype. send the data. */ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp; - - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete; - req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete; - req->kind = MPID_REQUEST_SEND; - - MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); - get_resp_pkt->request_handle = get_pkt->request_handle; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; - iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); - - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_pkt->addr; - MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); - iov[1].MPID_IOV_LEN = get_pkt->count * type_size; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPIU_Object_set_ref(req, 0); - MPIDI_CH3_Request_destroy(req); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - /* --END ERROR HANDLING-- */ - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - } - else { - /* derived datatype. first get the dtype_info and dataloop. */ - - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP_DERIVED_DT); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete; - req->dev.OnFinal = 0; - req->dev.user_buf = get_pkt->addr; - req->dev.user_count = get_pkt->count; - req->dev.datatype = MPI_DATATYPE_NULL; - req->dev.request_handle = get_pkt->request_handle; - - req->dev.dtype_info = (MPIDI_RMA_dtype_info *) - MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); - if (!req->dev.dtype_info) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_RMA_dtype_info"); - } - - req->dev.dataloop = MPIU_Malloc(get_pkt->dataloop_size); - if (!req->dev.dataloop) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - get_pkt->dataloop_size); - } - - /* if we received all of the dtype_info and dataloop, copy it - * now and call the handler, otherwise set the iov and let the - * channel copy it */ - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size) { - /* copy all of dtype_info and dataloop */ - MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); - MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - get_pkt->dataloop_size); - - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size; - - /* All dtype data has been received, call req handler */ - mpi_errno = MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete(vc, req, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET"); - if (complete) - *rreqp = NULL; - } - else { - req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; - req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); - req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = get_pkt->dataloop_size; - req->dev.iov_count = 2; - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = req; - } - - } - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET); - return mpi_errno; - -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_accum_t *accum_pkt = &pkt->accum; - MPID_Request *req = NULL; - MPI_Aint true_lb, true_extent, extent; - void *tmp_buf = NULL; - int complete = 0; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - MPID_Win *win_ptr; - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt"); - - MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags); - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); - *rreqp = req; - - req->dev.user_count = accum_pkt->count; - req->dev.op = accum_pkt->op; - req->dev.real_user_buf = accum_pkt->addr; - req->dev.target_win_handle = accum_pkt->target_win_handle; - req->dev.source_win_handle = accum_pkt->source_win_handle; - req->dev.flags = accum_pkt->flags; - - if (accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM) { - req->dev.resp_request_handle = accum_pkt->request_handle; - } - else { - req->dev.resp_request_handle = MPI_REQUEST_NULL; - } - - if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) { - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP); - req->dev.datatype = accum_pkt->datatype; - - MPIR_Type_get_true_extent_impl(accum_pkt->datatype, &true_lb, &true_extent); - MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent); - - /* Predefined types should always have zero lb */ - MPIU_Assert(true_lb == 0); - - tmp_buf = MPIU_Malloc(accum_pkt->count * (MPIR_MAX(extent, true_extent))); - if (!tmp_buf) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - accum_pkt->count * MPIR_MAX(extent, true_extent)); - } - - req->dev.user_buf = tmp_buf; - - MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size); - req->dev.recv_data_sz = type_size * accum_pkt->count; - - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); - /* FIXME: Only change the handling of completion if - * post_data_receive reset the handler. There should - * be a cleaner way to do this */ - if (!req->dev.OnDataAvail) { - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete; - } - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - - if (complete) { - mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete(vc, req, &complete); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } - } - else { - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete; - req->dev.datatype = MPI_DATATYPE_NULL; - req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutAccumRespComplete; - - req->dev.dtype_info = (MPIDI_RMA_dtype_info *) - MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); - if (!req->dev.dtype_info) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_RMA_dtype_info"); - } - - req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size); - if (!req->dev.dataloop) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - accum_pkt->dataloop_size); - } - - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size) { - /* copy all of dtype_info and dataloop */ - MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); - MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - accum_pkt->dataloop_size); - - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size; - - /* All dtype data has been received, call req handler */ - mpi_errno = MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete(vc, req, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_ACCUMULATE"); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } - else { - req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; - req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); - req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = accum_pkt->dataloop_size; - req->dev.iov_count = 2; - *buflen = sizeof(MPIDI_CH3_Pkt_t); - } - - } - - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); - return mpi_errno; - fn_fail: - goto fn_exit; - -} - -/* Special accumulate for short data items entirely within the packet */ -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Accumulate_Immed -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_accum_immed_t *accum_pkt = &pkt->accum_immed; - MPID_Win *win_ptr; - MPI_Aint extent; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate immedidate pkt"); - - MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags); - - /* return the number of bytes processed in this function */ - /* data_len == 0 (all within packet) */ - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - - MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent); - - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - /* Data is already present */ - if (accum_pkt->op == MPI_REPLACE) { - /* no datatypes required */ - int len; - MPIU_Assign_trunc(len, (accum_pkt->count * extent), int); - /* FIXME: use immediate copy because this is short */ - MPIUI_Memcpy(accum_pkt->addr, accum_pkt->data, len); - } - else { - if (HANDLE_GET_KIND(accum_pkt->op) == HANDLE_KIND_BUILTIN) { - MPI_User_function *uop; - /* get the function by indexing into the op table */ - uop = MPIR_OP_HDL_TO_FN(accum_pkt->op); - (*uop) (accum_pkt->data, accum_pkt->addr, &(accum_pkt->count), &(accum_pkt->datatype)); - } - else { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OP, "**opnotpredefined", - "**opnotpredefined %d", accum_pkt->op); - } - } - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - - /* There are additional steps to take if this is a passive - * target RMA or the last operation from the source */ - - /* Here is the code executed in PutAccumRespComplete after the - * accumulation operation */ - MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); - - mpi_errno = MPIDI_CH3_Finish_rma_op_target(vc, win_ptr, TRUE, - accum_pkt->flags, accum_pkt->source_win_handle); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); - return mpi_errno; - fn_fail: - goto fn_exit; - -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_CAS -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &upkt.cas_resp; - MPIDI_CH3_Pkt_cas_t *cas_pkt = &pkt->cas; - MPID_Win *win_ptr; - MPID_Request *req; - MPI_Aint len; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt"); - - MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, cas_pkt->flags); - - /* return the number of bytes processed in this function */ - /* data_len == 0 (all within packet) */ - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); - cas_resp_pkt->request_handle = cas_pkt->request_handle; - - /* Copy old value into the response packet */ - MPID_Datatype_get_size_macro(cas_pkt->datatype, len); - MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); - - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - - MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len); - - /* Compare and replace if equal */ - if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) { - MPIU_Memcpy(cas_pkt->addr, &cas_pkt->origin_data, len); - } - - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - - /* Send the response packet */ - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - - if (req != NULL) { - if (!MPID_Request_is_complete(req)) { - /* sending process is not completed, set proper OnDataAvail - (it is initialized to NULL by lower layer) */ - req->dev.target_win_handle = cas_pkt->target_win_handle; - req->dev.flags = cas_pkt->flags; - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetAccumRespComplete; - - /* here we increment the Active Target counter to guarantee the GET-like - operation are completed when counter reaches zero. */ - win_ptr->at_completion_counter++; - - MPID_Request_release(req); - goto fn_exit; - } - else - MPID_Request_release(req); - } - - - /* There are additional steps to take if this is a passive - * target RMA or the last operation from the source */ - - mpi_errno = MPIDI_CH3_Finish_rma_op_target(NULL, win_ptr, TRUE, cas_pkt->flags, MPI_WIN_NULL); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); - return mpi_errno; - fn_fail: - goto fn_exit; - -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_CASResp -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &pkt->cas_resp; - MPID_Request *req; - MPI_Aint len; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS response pkt"); - - MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); - MPID_Datatype_get_size_macro(req->dev.datatype, len); - - MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->data, len); - - MPIDI_CH3U_Request_complete(req); - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP); - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_FOP -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_fop_t *fop_pkt = &pkt->fop; - MPID_Request *req; - MPID_Win *win_ptr; - int data_complete = 0; - MPI_Aint len; - MPIU_CHKPMEM_DECL(1); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt"); - - MPIU_Assert(fop_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, fop_pkt->flags); - - req = MPID_Request_create(); - MPIU_ERR_CHKANDJUMP(req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); - MPIU_Object_set_ref(req, 1); /* Ref is held by progress engine */ - *rreqp = NULL; - - req->dev.user_buf = NULL; /* will be set later */ - req->dev.user_count = 1; - req->dev.datatype = fop_pkt->datatype; - req->dev.op = fop_pkt->op; - req->dev.real_user_buf = fop_pkt->addr; - req->dev.target_win_handle = fop_pkt->target_win_handle; - req->dev.request_handle = fop_pkt->request_handle; - req->dev.flags = fop_pkt->flags; - - MPID_Datatype_get_size_macro(req->dev.datatype, len); - MPIU_Assert(len <= sizeof(MPIDI_CH3_FOP_Immed_u)); - - /* Set up the user buffer and receive data if needed */ - if (len <= sizeof(fop_pkt->origin_data) || fop_pkt->op == MPI_NO_OP) { - req->dev.user_buf = fop_pkt->origin_data; - *buflen = sizeof(MPIDI_CH3_Pkt_t); - data_complete = 1; - } - else { - /* Data won't fit in the header, allocate temp space and receive it */ - MPIDI_msg_sz_t data_len; - void *data_buf; - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - req->dev.recv_data_sz = len; /* count == 1 for FOP */ - - MPIU_CHKPMEM_MALLOC(req->dev.user_buf, void *, len, mpi_errno, "**nomemreq"); - - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &data_complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); - - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPComplete; - - if (!data_complete) { - *rreqp = req; - } - - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - } - - if (data_complete) { - int fop_complete = 0; - mpi_errno = MPIDI_CH3_ReqHandler_FOPComplete(vc, req, &fop_complete); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - *rreqp = NULL; - } - - fn_exit: - MPIU_CHKPMEM_COMMIT(); - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - MPIU_CHKPMEM_REAP(); - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_FOPResp -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &pkt->fop_resp; - MPID_Request *req; - int complete = 0; - MPI_Aint len; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP response pkt"); - - MPID_Request_get_ptr(fop_resp_pkt->request_handle, req); - MPID_Datatype_get_size_macro(req->dev.datatype, len); - - if (len <= sizeof(fop_resp_pkt->data)) { - MPIU_Memcpy(req->dev.user_buf, (void *) fop_resp_pkt->data, len); - *buflen = sizeof(MPIDI_CH3_Pkt_t); - complete = 1; - } - else { - /* Data was too big to embed in the header */ - MPIDI_msg_sz_t data_len; - void *data_buf; - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - req->dev.recv_data_sz = len; /* count == 1 for FOP */ - *rreqp = req; - - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_RESP"); - - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - } - - if (complete) { - MPIDI_CH3U_Request_complete(req); - *rreqp = NULL; - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Get_AccumResp -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &pkt->get_accum_resp; - MPID_Request *req; - int complete; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt"); - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req); - - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); - req->dev.recv_data_sz = type_size * req->dev.user_count; - - *rreqp = req; - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_ACCUM_RESP"); - if (complete) { - MPIDI_CH3U_Request_complete(req); - *rreqp = NULL; - } - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Lock -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt->lock; - MPID_Win *win_ptr = NULL; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock pkt"); - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr); - - if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_pkt->lock_type) == 1) { - /* send lock granted packet. */ - mpi_errno = MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr, lock_pkt->source_win_handle); - } - - else { - /* queue the lock information */ - MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; - - /* Note: This code is reached by the fechandadd rma tests */ - /* FIXME: MT: This may need to be done atomically. */ - - /* FIXME: Since we need to add to the tail of the list, - * we should maintain a tail pointer rather than traversing the - * list each time to find the tail. */ - curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; - prev_ptr = curr_ptr; - while (curr_ptr != NULL) { - prev_ptr = curr_ptr; - curr_ptr = curr_ptr->next; - } - - new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - if (!new_ptr) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_Win_lock_queue"); - } - if (prev_ptr != NULL) - prev_ptr->next = new_ptr; - else - win_ptr->lock_queue = new_ptr; - - new_ptr->next = NULL; - new_ptr->lock_type = lock_pkt->lock_type; - new_ptr->source_win_handle = lock_pkt->source_win_handle; - new_ptr->vc = vc; - new_ptr->pt_single_op = NULL; - } - - *rreqp = NULL; - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_LockPutUnlock -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_lock_put_unlock_t *lock_put_unlock_pkt = &pkt->lock_put_unlock; - MPID_Win *win_ptr = NULL; - MPID_Request *req = NULL; - MPI_Aint type_size; - int complete; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - int mpi_errno = MPI_SUCCESS; - int (*fcn)( MPIDI_VC_t *, struct MPID_Request *, int * ); - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_put_unlock pkt"); - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); - - req->dev.datatype = lock_put_unlock_pkt->datatype; - MPID_Datatype_get_size_macro(lock_put_unlock_pkt->datatype, type_size); - req->dev.recv_data_sz = type_size * lock_put_unlock_pkt->count; - req->dev.user_count = lock_put_unlock_pkt->count; - req->dev.target_win_handle = lock_put_unlock_pkt->target_win_handle; - - MPID_Win_get_ptr(lock_put_unlock_pkt->target_win_handle, win_ptr); - - if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_put_unlock_pkt->lock_type) == 1) { - /* do the put. for this optimization, only basic datatypes supported. */ - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete; - req->dev.user_buf = lock_put_unlock_pkt->addr; - req->dev.source_win_handle = lock_put_unlock_pkt->source_win_handle; - req->dev.flags = lock_put_unlock_pkt->flags; - } - - else { - /* queue the information */ - MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; - - new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - if (!new_ptr) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_Win_lock_queue"); - } - - new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op)); - if (new_ptr->pt_single_op == NULL) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_PT_single_op"); - } - - /* FIXME: MT: The queuing may need to be done atomically. */ - - curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; - prev_ptr = curr_ptr; - while (curr_ptr != NULL) { - prev_ptr = curr_ptr; - curr_ptr = curr_ptr->next; - } - - if (prev_ptr != NULL) - prev_ptr->next = new_ptr; - else - win_ptr->lock_queue = new_ptr; - - new_ptr->next = NULL; - new_ptr->lock_type = lock_put_unlock_pkt->lock_type; - new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle; - new_ptr->vc = vc; - - new_ptr->pt_single_op->type = MPIDI_RMA_PUT; - new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags; - new_ptr->pt_single_op->addr = lock_put_unlock_pkt->addr; - new_ptr->pt_single_op->count = lock_put_unlock_pkt->count; - new_ptr->pt_single_op->datatype = lock_put_unlock_pkt->datatype; - /* allocate memory to receive the data */ - new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz); - if (new_ptr->pt_single_op->data == NULL) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - req->dev.recv_data_sz); - } - - new_ptr->pt_single_op->data_recd = 0; - - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_PUT); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete; - req->dev.user_buf = new_ptr->pt_single_op->data; - req->dev.lock_queue_entry = new_ptr; - } - - fcn = req->dev.OnDataAvail; - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, - "**ch3|postrecv", "**ch3|postrecv %s", - "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK"); - } - req->dev.OnDataAvail = fcn; - *rreqp = req; - - if (complete) { - mpi_errno = fcn(vc, req, &complete); - if (complete) { - *rreqp = NULL; - } - } - - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, - "**ch3|postrecv", "**ch3|postrecv %s", - "MPIDI_CH3_PKT_LOCK_PUT_UNLOCK"); - } - - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKPUTUNLOCK); - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_LockGetUnlock -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_lock_get_unlock_t *lock_get_unlock_pkt = &pkt->lock_get_unlock; - MPID_Win *win_ptr = NULL; - MPI_Aint type_size; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_get_unlock pkt"); - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - MPID_Win_get_ptr(lock_get_unlock_pkt->target_win_handle, win_ptr); - - if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_get_unlock_pkt->lock_type) == 1) { - /* do the get. for this optimization, only basic datatypes supported. */ - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp; - MPID_Request *req; - MPID_IOV iov[MPID_IOV_LIMIT]; - - req = MPID_Request_create(); - req->dev.target_win_handle = lock_get_unlock_pkt->target_win_handle; - req->dev.source_win_handle = lock_get_unlock_pkt->source_win_handle; - req->dev.flags = lock_get_unlock_pkt->flags; - - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete; - req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendRespComplete; - req->kind = MPID_REQUEST_SEND; - - /* here we increment the Active Target counter to guarantee the GET-like - operation are completed when counter reaches zero. */ - win_ptr->at_completion_counter++; - - MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); - get_resp_pkt->request_handle = lock_get_unlock_pkt->request_handle; - - iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; - iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); - - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) lock_get_unlock_pkt->addr; - MPID_Datatype_get_size_macro(lock_get_unlock_pkt->datatype, type_size); - iov[1].MPID_IOV_LEN = lock_get_unlock_pkt->count * type_size; - - mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPIU_Object_set_ref(req, 0); - MPIDI_CH3_Request_destroy(req); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - } - /* --END ERROR HANDLING-- */ - } - - else { - /* queue the information */ - MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; - - /* FIXME: MT: This may need to be done atomically. */ - - curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; - prev_ptr = curr_ptr; - while (curr_ptr != NULL) { - prev_ptr = curr_ptr; - curr_ptr = curr_ptr->next; - } - - new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - if (!new_ptr) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_Win_lock_queue"); - } - new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op)); - if (new_ptr->pt_single_op == NULL) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_PT_Single_op"); - } - - if (prev_ptr != NULL) - prev_ptr->next = new_ptr; - else - win_ptr->lock_queue = new_ptr; - - new_ptr->next = NULL; - new_ptr->lock_type = lock_get_unlock_pkt->lock_type; - new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle; - new_ptr->vc = vc; - - new_ptr->pt_single_op->type = MPIDI_RMA_GET; - new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags; - new_ptr->pt_single_op->addr = lock_get_unlock_pkt->addr; - new_ptr->pt_single_op->count = lock_get_unlock_pkt->count; - new_ptr->pt_single_op->datatype = lock_get_unlock_pkt->datatype; - new_ptr->pt_single_op->data = NULL; - new_ptr->pt_single_op->request_handle = lock_get_unlock_pkt->request_handle; - new_ptr->pt_single_op->data_recd = 1; - } - - *rreqp = NULL; - - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGETUNLOCK); - return mpi_errno; -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_LockAccumUnlock -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_lock_accum_unlock_t *lock_accum_unlock_pkt = &pkt->lock_accum_unlock; - MPID_Request *req = NULL; - MPID_Win *win_ptr = NULL; - MPIDI_Win_lock_queue *curr_ptr = NULL, *prev_ptr = NULL, *new_ptr = NULL; - MPI_Aint type_size; - int complete; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock_accum_unlock pkt"); - - /* no need to acquire the lock here because we need to receive the - * data into a temporary buffer first */ - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); - - req->dev.datatype = lock_accum_unlock_pkt->datatype; - MPID_Datatype_get_size_macro(lock_accum_unlock_pkt->datatype, type_size); - req->dev.recv_data_sz = type_size * lock_accum_unlock_pkt->count; - req->dev.user_count = lock_accum_unlock_pkt->count; - req->dev.target_win_handle = lock_accum_unlock_pkt->target_win_handle; - req->dev.flags = lock_accum_unlock_pkt->flags; - - /* queue the information */ - - new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - if (!new_ptr) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_Win_lock_queue"); - } - - new_ptr->pt_single_op = (MPIDI_PT_single_op *) MPIU_Malloc(sizeof(MPIDI_PT_single_op)); - if (new_ptr->pt_single_op == NULL) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", - "MPIDI_PT_single_op"); - } - - MPID_Win_get_ptr(lock_accum_unlock_pkt->target_win_handle, win_ptr); - - /* FIXME: MT: The queuing may need to be done atomically. */ - - curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue; - prev_ptr = curr_ptr; - while (curr_ptr != NULL) { - prev_ptr = curr_ptr; - curr_ptr = curr_ptr->next; - } - - if (prev_ptr != NULL) - prev_ptr->next = new_ptr; - else - win_ptr->lock_queue = new_ptr; - - new_ptr->next = NULL; - new_ptr->lock_type = lock_accum_unlock_pkt->lock_type; - new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle; - new_ptr->vc = vc; - - new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE; - new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags; - new_ptr->pt_single_op->addr = lock_accum_unlock_pkt->addr; - new_ptr->pt_single_op->count = lock_accum_unlock_pkt->count; - new_ptr->pt_single_op->datatype = lock_accum_unlock_pkt->datatype; - new_ptr->pt_single_op->op = lock_accum_unlock_pkt->op; - /* allocate memory to receive the data */ - new_ptr->pt_single_op->data = MPIU_Malloc(req->dev.recv_data_sz); - if (new_ptr->pt_single_op->data == NULL) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - req->dev.recv_data_sz); - } - - new_ptr->pt_single_op->data_recd = 0; - - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PT_SINGLE_ACCUM); - req->dev.user_buf = new_ptr->pt_single_op->data; - req->dev.lock_queue_entry = new_ptr; - - *rreqp = req; - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - /* FIXME: Only change the handling of completion if - * post_data_receive reset the handler. There should - * be a cleaner way to do this */ - if (!req->dev.OnDataAvail) { - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SinglePutAccumComplete; - } - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK"); - } - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - - if (complete) { - mpi_errno = MPIDI_CH3_ReqHandler_SinglePutAccumComplete(vc, req, &complete); - if (complete) { - *rreqp = NULL; - } - } - fn_fail: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKACCUMUNLOCK); - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_GetResp -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &pkt->get_resp; - MPID_Request *req; - int complete; - char *data_buf = NULL; - MPIDI_msg_sz_t data_len; - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get response pkt"); - - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - MPID_Request_get_ptr(get_resp_pkt->request_handle, req); - - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); - req->dev.recv_data_sz = type_size * req->dev.user_count; - - *rreqp = req; - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", - "MPIDI_CH3_PKT_GET_RESP"); - if (complete) { - MPIDI_CH3U_Request_complete(req); - *rreqp = NULL; - } - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_LockGranted -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_lock_granted_t *lock_granted_pkt = &pkt->lock_granted; - MPID_Win *win_ptr = NULL; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock granted pkt"); - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - MPID_Win_get_ptr(lock_granted_pkt->source_win_handle, win_ptr); - /* set the remote_lock_state flag in the window */ - win_ptr->targets[lock_granted_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; - - *rreqp = NULL; - MPIDI_CH3_Progress_signal_completion(); - - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED); - return MPI_SUCCESS; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_PtRMADone -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_PtRMADone(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &pkt->pt_rma_done; - MPID_Win *win_ptr = NULL; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt"); - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - MPID_Win_get_ptr(pt_rma_done_pkt->source_win_handle, win_ptr); - MPIU_Assert(win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state != - MPIDI_CH3_WIN_LOCK_NONE); - - if (win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state == - MPIDI_CH3_WIN_LOCK_FLUSH) - win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = - MPIDI_CH3_WIN_LOCK_GRANTED; - else - win_ptr->targets[pt_rma_done_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE; - - *rreqp = NULL; - MPIDI_CH3_Progress_signal_completion(); - - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PTRMADONE); - return MPI_SUCCESS; -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Unlock -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Unlock(MPIDI_VC_t * vc ATTRIBUTE((unused)), - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_unlock_t *unlock_pkt = &pkt->unlock; - MPID_Win *win_ptr = NULL; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK); - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received unlock pkt"); - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - - MPID_Win_get_ptr(unlock_pkt->target_win_handle, win_ptr); - mpi_errno = MPIDI_CH3I_Release_lock(win_ptr); - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - MPIDI_CH3_Progress_signal_completion(); - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_UNLOCK); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_PktHandler_Flush -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_PktHandler_Flush(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_CH3_Pkt_flush_t *flush_pkt = &pkt->flush; - MPID_Win *win_ptr = NULL; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH); - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received flush pkt"); - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - *rreqp = NULL; - - /* This is a flush request packet */ - if (flush_pkt->target_win_handle != MPI_WIN_NULL) { - MPID_Request *req = NULL; - - MPID_Win_get_ptr(flush_pkt->target_win_handle, win_ptr); - - flush_pkt->target_win_handle = MPI_WIN_NULL; - flush_pkt->target_rank = win_ptr->comm_ptr->rank; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - /* Release the request returned by iStartMsg */ - if (req != NULL) { - MPID_Request_release(req); - } - } - - /* This is a flush response packet */ - else { - MPID_Win_get_ptr(flush_pkt->source_win_handle, win_ptr); - win_ptr->targets[flush_pkt->target_rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; - MPIDI_CH3_Progress_signal_completion(); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FLUSH); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -/* ------------------------------------------------------------------------ */ /* list_complete_timer/counter and list_block_timer defined above */ -static inline int poke_progress_engine(void) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Progress_state progress_state; - - MPID_Progress_start(&progress_state); - mpi_errno = MPID_Progress_poke(); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPID_Progress_end(&progress_state); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * ops_list) { int ntimes = 0, mpi_errno = 0; @@ -5085,6 +2491,7 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o /* In some tests, this hung unless the test ensured that * there was an incomplete request. */ curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); + /* MT: avoid processing unissued operations enqueued by other threads in MPID_Progress_wait() */ if (curr_ptr && !curr_ptr->request) { @@ -5187,271 +2594,3 @@ static inline int rma_list_gc(MPID_Win * win_ptr, fn_fail: return mpi_errno; } - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_Start_rma_op_target -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_Start_rma_op_target(MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET); - - /* Lock with NOCHECK is piggybacked on this message. We should be able to - * immediately grab the lock. Otherwise, there is a synchronization error. */ - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK && flags & MPIDI_CH3_PKT_FLAG_RMA_NOCHECK) { - int lock_acquired; - int lock_mode; - - if (flags & MPIDI_CH3_PKT_FLAG_RMA_SHARED) { - lock_mode = MPI_LOCK_SHARED; - } - else if (flags & MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE) { - lock_mode = MPI_LOCK_EXCLUSIVE; - } - else { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|rma_flags"); - } - - lock_acquired = MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_mode); - MPIU_ERR_CHKANDJUMP(!lock_acquired, mpi_errno, MPI_ERR_RMA_SYNC, "**ch3|nocheck_invalid"); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_START_RMA_OP_TARGET); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3_Finish_rma_op_target -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_rma_update, - MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET); - - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET); - - /* This function should be called by the target process after each RMA - * operation is completed, to update synchronization state. */ - - /* Last RMA operation from source. If active target RMA, decrement window - * counter. */ - if (flags & MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE) { - MPIU_Assert(win_ptr->current_lock_type == MPID_LOCK_NONE); - - win_ptr->at_completion_counter -= 1; - MPIU_Assert(win_ptr->at_completion_counter >= 0); - - /* Signal the local process when the op counter reaches 0. */ - if (win_ptr->at_completion_counter == 0) - MPIDI_CH3_Progress_signal_completion(); - } - - /* If passive target RMA, release lock on window and grant next lock in the - * lock queue if there is any. If requested by the origin, send an ack back - * to indicate completion at the target. */ - else if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { - MPIU_Assert(win_ptr->current_lock_type != MPID_LOCK_NONE); - - if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) { - MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL); - mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr, source_win_handle); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } - - mpi_errno = MPIDI_CH3I_Release_lock(win_ptr); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - /* The local process may be waiting for the lock. Signal completion to - * wake it up, so it can attempt to grab the lock. */ - MPIDI_CH3_Progress_signal_completion(); - } - else if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) { - /* Ensure store instructions have been performed before flush call is - * finished on origin process. */ - OPA_read_write_barrier(); - - if (flags & MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK) { - MPIDI_CH3_Pkt_t upkt; - MPIDI_CH3_Pkt_flush_t *flush_pkt = &upkt.flush; - MPID_Request *req = NULL; - - MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received piggybacked flush request"); - MPIU_Assert(source_win_handle != MPI_WIN_NULL && vc != NULL); - - MPIDI_Pkt_init(flush_pkt, MPIDI_CH3_PKT_FLUSH); - flush_pkt->source_win_handle = source_win_handle; - flush_pkt->target_win_handle = MPI_WIN_NULL; - flush_pkt->target_rank = win_ptr->comm_ptr->rank; - - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, flush_pkt, sizeof(*flush_pkt), &req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); - MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**ch3|rma_msg"); - - /* Release the request returned by iStartMsg */ - if (req != NULL) { - MPID_Request_release(req); - } - } - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_FINISH_RMA_OP_TARGET); - return mpi_errno; - /* --BEGIN ERROR HANDLING-- */ - fn_fail: - goto fn_exit; - /* --END ERROR HANDLING-- */ -} - - -/* ------------------------------------------------------------------------ */ -/* - * For debugging, we provide the following functions for printing the - * contents of an RMA packet - */ -/* ------------------------------------------------------------------------ */ -#ifdef MPICH_DBG_OUTPUT -int MPIDI_CH3_PktPrint_Put(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PUT\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->put.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->put.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->put.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.dataloop_size)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->put.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->put.source_win_handle)); - /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->put.win_ptr)); */ - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_Get(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->get.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->get.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->get.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.dataloop_size)); - MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request_handle)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->get.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->get.source_win_handle)); - /* - * MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request)); - * MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->get.win_ptr)); - */ - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_GetResp(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_GET_RESP\n")); - MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request_handle)); - /*MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get_resp.request)); */ - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_Accumulate(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUMULATE\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.dataloop_size)); - MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum.op)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum.source_win_handle)); - /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */ - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_Accum_Immed(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_ACCUM_IMMED\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum_immed.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum_immed.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum_immed.datatype)); - MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum_immed.op)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum_immed.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum_immed.source_win_handle)); - /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->accum.win_ptr)); */ - fflush(stdout); - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_Lock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK\n")); - MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock.lock_type)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock.source_win_handle)); - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_LockPutUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_PUT_UNLOCK\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_put_unlock.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_put_unlock.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_put_unlock.datatype)); - MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_put_unlock.lock_type)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_put_unlock.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_put_unlock.source_win_handle)); - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_LockAccumUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_accum_unlock.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_accum_unlock.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_accum_unlock.datatype)); - MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_accum_unlock.lock_type)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_accum_unlock.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle)); - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_LockGetUnlock(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GET_UNLOCK\n")); - MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->lock_get_unlock.addr)); - MPIU_DBG_PRINTF((" count ........ %d\n", pkt->lock_get_unlock.count)); - MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->lock_get_unlock.datatype)); - MPIU_DBG_PRINTF((" lock_type .... %d\n", pkt->lock_get_unlock.lock_type)); - MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->lock_get_unlock.target_win_handle)); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_get_unlock.source_win_handle)); - MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->lock_get_unlock.request_handle)); - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_PtRMADone(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_PT_RMA_DONE\n")); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_accum_unlock.source_win_handle)); - return MPI_SUCCESS; -} - -int MPIDI_CH3_PktPrint_LockGranted(FILE * fp, MPIDI_CH3_Pkt_t * pkt) -{ - MPIU_DBG_PRINTF((" type ......... MPIDI_CH3_PKT_LOCK_GRANTED\n")); - MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->lock_granted.source_win_handle)); - return MPI_SUCCESS; -} -#endif diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c new file mode 100644 index 0000000..5a7221c --- /dev/null +++ b/src/mpid/ch3/src/mpidi_rma.c @@ -0,0 +1,128 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#include "mpidimpl.h" +#include "mpidrma.h" + + +#undef FUNCNAME +#define FUNCNAME MPIDI_Win_free +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_Win_free(MPID_Win ** win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + int in_use; + MPID_Comm *comm_ptr; + int errflag = FALSE; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FREE); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FREE); + + MPIU_ERR_CHKANDJUMP((*win_ptr)->epoch_state != MPIDI_EPOCH_NONE, + mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); + + if (!(*win_ptr)->shm_allocated) { + /* when SHM is allocated, we already did a global barrier in + MPIDI_CH3_SHM_Win_free, so we do not need to do it again here. */ + mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + } + + comm_ptr = (*win_ptr)->comm_ptr; + mpi_errno = MPIR_Comm_free_impl(comm_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + + MPIU_Free((*win_ptr)->targets); + MPIU_Free((*win_ptr)->base_addrs); + MPIU_Free((*win_ptr)->sizes); + MPIU_Free((*win_ptr)->disp_units); + MPIU_Free((*win_ptr)->all_win_handles); + + /* Free the attached buffer for windows created with MPI_Win_allocate() */ + if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE || + (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED) { + if ((*win_ptr)->shm_allocated == FALSE && (*win_ptr)->size > 0) { + MPIU_Free((*win_ptr)->base); + } + } + + MPIU_Object_release_ref(*win_ptr, &in_use); + /* MPI windows don't have reference count semantics, so this should always be true */ + MPIU_Assert(!in_use); + MPIU_Handle_obj_free(&MPID_Win_mem, *win_ptr); + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FREE); + return mpi_errno; + + fn_fail: + goto fn_exit; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_Win_shared_query +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_Win_shared_query(MPID_Win * win_ptr, int target_rank, MPI_Aint * size, + int *disp_unit, void *baseptr) +{ + int mpi_errno = MPI_SUCCESS; + + MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_SHARED_QUERY); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_SHARED_QUERY); + + *(void **) baseptr = win_ptr->base; + *size = win_ptr->size; + *disp_unit = win_ptr->disp_unit; + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_SHARED_QUERY); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_Alloc_mem +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +void *MPIDI_Alloc_mem(size_t size, MPID_Info * info_ptr) +{ + void *ap; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_ALLOC_MEM); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_ALLOC_MEM); + + ap = MPIU_Malloc(size); + + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_ALLOC_MEM); + return ap; +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_Free_mem +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_Free_mem(void *ptr) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPIDI_FREE_MEM); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_FREE_MEM); + + MPIU_Free(ptr); + + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_FREE_MEM); + return mpi_errno; +} http://git.mpich.org/mpich.git/commitdiff/5c51303260595939cb15d658e5f369bc1… commit 5c51303260595939cb15d658e5f369bc148c9a4a Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Nov 2 20:10:48 2014 -0600 Temporarily remove all RMA PVARs. Because we are going to rewrite the RMA infrastructure and many PVARs will no longer be used, here we temporarily remove all PVARs and will add needed PVARs back after new implementation is done. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index c786dcb..b28d50c 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -14,8 +14,6 @@ #define MPIDI_CH3_PAGESIZE_MASK (~(MPIDI_CH3_PAGESIZE-1)) #define MPIDI_CH3_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_CH3_PAGESIZE_MASK)) & MPIDI_CH3_PAGESIZE_MASK) -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather); - MPIDI_SHM_Wins_list_t shm_wins_list; static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, @@ -317,7 +315,6 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * node_size = node_comm_ptr->local_size; node_rank = node_comm_ptr->rank; - MPIR_T_PVAR_TIMER_START(RMA, rma_wincreate_allgather); /* allocate memory for the base addresses, disp_units, and completion counters of all processes */ MPIU_CHKPMEM_MALLOC((*win_ptr)->base_addrs, void **, @@ -350,7 +347,6 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, tmp_buf, 3 * sizeof(MPI_Aint), MPI_BYTE, (*win_ptr)->comm_ptr, &errflag); - MPIR_T_PVAR_TIMER_END(RMA, rma_wincreate_allgather); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 652af02..3710086 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -9,10 +9,6 @@ #include "mpl_utlist.h" #include "mpidi_ch3_impl.h" -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_rs); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_complete); - typedef enum MPIDI_RMA_Op_type { MPIDI_RMA_PUT = 23, MPIDI_RMA_GET = 24, diff --git a/src/mpid/ch3/src/ch3u_rma_acc_ops.c b/src/mpid/ch3/src/ch3u_rma_acc_ops.c index a7edf6a..5d72483 100644 --- a/src/mpid/ch3/src/ch3u_rma_acc_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_acc_ops.c @@ -6,9 +6,6 @@ #include "mpidrma.h" -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set); - #undef FUNCNAME #define FUNCNAME MPIDI_Get_accumulate #undef FCNAME @@ -79,16 +76,13 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, MPIDI_RMA_Op_t *new_ptr = NULL; /* Append the operation to the window's RMA ops queue */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); new_ptr->type = MPIDI_RMA_GET_ACCUMULATE; /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t * contain both PUT and GET like ops */ @@ -103,7 +97,6 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->target_count = target_count; new_ptr->target_datatype = target_datatype; new_ptr->op = op; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); /* if source or target datatypes are derived, increment their * reference counts */ @@ -193,14 +186,11 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIDI_RMA_Op_t *new_ptr = NULL; /* Append this operation to the RMA ops queue */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP; new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; @@ -215,7 +205,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, new_ptr->compare_addr = (void *) compare_addr; new_ptr->compare_count = 1; new_ptr->compare_datatype = datatype; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); } fn_exit: @@ -288,14 +277,11 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIDI_RMA_Op_t *new_ptr = NULL; /* Append this operation to the RMA ops queue */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); new_ptr->type = MPIDI_RMA_FETCH_AND_OP; new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; @@ -308,7 +294,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, new_ptr->result_count = 1; new_ptr->result_datatype = datatype; new_ptr->op = op; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); } fn_exit: diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 83b1e37..d91a626 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -8,9 +8,6 @@ static int enableShortACC = 1; -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set); - #define MPIDI_PASSIVE_TARGET_DONE_TAG 348297 #define MPIDI_PASSIVE_TARGET_RMA_TAG 563924 @@ -180,14 +177,11 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIDI_RMA_Op_t *new_ptr = NULL; /* queue it up */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->type = MPIDI_RMA_PUT; /* Cast away const'ness for the origin address, as the @@ -200,7 +194,6 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->target_disp = target_disp; new_ptr->target_count = target_count; new_ptr->target_datatype = target_datatype; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); /* if source or target datatypes are derived, increment their * reference counts */ @@ -291,14 +284,11 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype MPIDI_RMA_Op_t *new_ptr = NULL; /* queue it up */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->type = MPIDI_RMA_GET; new_ptr->origin_addr = origin_addr; @@ -308,7 +298,6 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->target_disp = target_disp; new_ptr->target_count = target_count; new_ptr->target_datatype = target_datatype; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); /* if source or target datatypes are derived, increment their * reference counts */ @@ -400,9 +389,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype MPIDI_RMA_Op_t *new_ptr = NULL; /* queue it up */ - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -410,7 +397,6 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype /* If predefined and contiguous, use a simplified element */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && enableShortACC) { - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); new_ptr->type = MPIDI_RMA_ACC_CONTIG; /* Only the information needed for the contig/predefined acc */ /* Cast away const'ness for origin_address as @@ -423,11 +409,9 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->target_count = target_count; new_ptr->target_datatype = target_datatype; new_ptr->op = op; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); goto fn_exit; } - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); new_ptr->type = MPIDI_RMA_ACCUMULATE; /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t * contain both PUT and GET like ops */ @@ -439,7 +423,6 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->target_count = target_count; new_ptr->target_datatype = target_datatype; new_ptr->op = op; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); /* if source or target datatypes are derived, increment their * reference counts */ diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 416f2f5..489309a 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -334,547 +334,8 @@ cvars: PROC_SYNC with origin will see the latest data. */ -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_lockqueue_alloc); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_clearlock); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_clearlock_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_rs); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_issue); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_issue_aux); -MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(RMA, rma_winfence_issue_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_complete); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_complete_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_wait); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_wait_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_block); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_block_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winpost_clearlock); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winpost_clearlock_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winpost_sendsync); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winpost_sendsync_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winstart_clearlock); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winstart_clearlock_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_issue); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_issue_aux); -MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(RMA, rma_wincomplete_issue_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_complete); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_complete_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_recvsync); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_recvsync_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincomplete_block); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_block_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winwait_wait); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winwait_wait_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winlock_getlocallock); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_getlock); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_issue); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_complete); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winunlock_complete_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winunlock_block); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winunlock_block_aux); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc_predef); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc_immed); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc_immed_op); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_cas); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_fop); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_get_accum); - -MPIR_T_PVAR_ULONG2_LEVEL_DECL(RMA, rma_winfence_reqs); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winfence_reqs); -MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(RMA, rma_winfence_reqs); - -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_winunlock_reqs); -MPIR_T_PVAR_ULONG2_COUNTER_DECL(RMA, rma_wincomplete_reqs); - -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_wincreate_allgather); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfree_rs); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfree_complete); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmaqueue_alloc); -MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmaqueue_set); - void MPIDI_CH3_RMA_Init_Pvars(void) { - /* rma_lockqueue_alloc */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_lockqueue_alloc, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "Allocate Lock Queue element (in seconds)"); - - /* rma_winfence_clearlock */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfence_clearlock, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Clear prior lock (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winfence_clearlock_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Clear prior lock"); - - /* rma_winfence_rs */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfence_rs, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:ReduceScatterBlock (in seconds)"); - - /* rma_winfence_issue and auxiliaries */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfence_issue, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Issue RMA ops (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winfence_issue_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Issue RMA ops"); - - MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_winfence_issue_aux, 0, /* init value */ - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Issue RMA ops"); - - /* rma_winfence_complete */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfence_complete, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Complete RMA ops (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winfence_complete_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Complete RMA ops"); - - /* rma_winfence_wait */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfence_wait, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", - "WIN_FENCE:Wait for ops from other processes (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winfence_wait_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Wait for ops from other processes"); - - /* rma_winfence_block */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfence_block, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Wait for any progress (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winfence_block_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Wait for any progress"); - - - /* rma_winpost_clearlock */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winpost_clearlock, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_POST:Clear prior lock (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winpost_clearlock_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_POST:Clear prior lock"); - - /* rma_winpost_sendsync */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winpost_sendsync, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_POST:Senc sync messages (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winpost_sendsync_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_POST:Senc sync messages"); - - /* rma_winstart_clearlock */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winstart_clearlock, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_START:Clear prior lock (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winstart_clearlock_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_START:Clear prior lock"); - - /* rma_wincomplete_issue and auxiliaries */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_wincomplete_issue, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Issue RMA ops (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_wincomplete_issue_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Issue RMA ops"); - - MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_wincomplete_issue_aux, 0, /* init value */ - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Issue RMA ops"); - - /* rma_wincomplete_complete */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_wincomplete_complete, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Complete RMA ops (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_wincomplete_complete_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Complete RMA ops"); - - /* rma_wincomplete_recvsync */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_wincomplete_recvsync, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Recv sync messages (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_wincomplete_recvsync_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Recv sync messages"); - - /* rma_wincomplete_block */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_wincomplete_block, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Wait for any progress (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_wincomplete_block_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Wait for any progress"); - - /* rma_winwait_wait */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winwait_wait, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", - "WIN_WAIT:Wait for ops from other processes (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winwait_wait_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_WAIT:Wait for ops from other processes"); - - /* rma_winlock_getlocallock */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winlock_getlocallock, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_LOCK:Get local lock (in seconds)"); - - /* rma_winunlock_getlock */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winunlock_getlock, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Acquire lock (in seconds)"); - - /* rma_winunlock_issue */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winunlock_issue, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Issue RMA ops (in seconds)"); - - /* rma_winunlock_complete */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winunlock_complete, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Complete RMA ops (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winunlock_complete_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Complete RMA ops (in seconds)"); - - /* rma_winunlock_block */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winunlock_block, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Wait for any progress (in seconds)"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winunlock_block_aux, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Wait for any progress"); - - /* rma_rmapkt_acc */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_acc, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "RMA:PKTHANDLER for Accumulate (in seconds)"); - - /* rma_rmapkt_acc_predef */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_acc_predef, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", - "RMA:PKTHANDLER for Accumulate: predef dtype (in seconds)"); - - /* rma_rmapkt_acc_immed */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_acc_immed, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "RMA:PKTHANDLER for Accum immed (in seconds)"); - - /* rma_rmapkt_acc_immed_op */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_acc_immed_op, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", - "RMA:PKTHANDLER for Accum immed operation (in seconds)"); - - /* rma_rmapkt_cas */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_cas, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "RMA:PKTHANDLER for Compare-and-swap (in seconds)"); - - /* rma_rmapkt_fop */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_fop, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "RMA:PKTHANDLER for Fetch-and-op (in seconds)"); - - /* rma_rmapkt_get_accum */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmapkt_get_accum, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "RMA:PKTHANDLER for Get-Accumulate (in seconds)"); - - /* Level, counter and highwatermark for rma_winfence_reqs */ - MPIR_T_PVAR_LEVEL_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_winfence_reqs, 0, /* init value */ - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS, - "RMA", "WIN_FENCE:Pending requests"); - - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winfence_reqs, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Pending requests"); - - MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(RMA, MPI_UNSIGNED_LONG_LONG, rma_winfence_reqs, 0, /* init value */ - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FENCE:Pending requests"); - - /* rma_winunlock_reqs */ - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_winunlock_reqs, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_UNLOCK:Pending requests"); - - /* rma_wincomplete_reqs */ - MPIR_T_PVAR_COUNTER_REGISTER_STATIC(RMA, - MPI_UNSIGNED_LONG_LONG, - rma_wincomplete_reqs, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_COMPLETE:Pending requests"); - - /* rma_wincreate_allgather */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_wincreate_allgather, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_CREATE:Allgather (in seconds)"); - - /* rma_winfree_rs */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfree_rs, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FREE:ReduceScatterBlock (in seconds)"); - - /* rma_winfree_complete */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_winfree_complete, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "WIN_FREE:Complete (in seconds)"); - - /* rma_rmaqueue_alloc */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmaqueue_alloc, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "Allocate RMA Queue element (in seconds)"); - - /* rma_rmaqueue_set */ - MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA, - MPI_DOUBLE, - rma_rmaqueue_set, - MPI_T_VERBOSITY_MPIDEV_DETAIL, - MPI_T_BIND_NO_OBJECT, - MPIR_T_PVAR_FLAG_READONLY, - "RMA", "Set fields in RMA Queue element (in seconds)"); } /* These are used to use a common routine to complete lists of RMA @@ -883,9 +344,6 @@ void MPIDI_CH3_RMA_Init_Pvars(void) thread-safe; the best choice for thread-safety is to eliminate this ability to discriminate between the different types of RMA synchronization. */ -static MPIR_T_pvar_timer_t *list_complete_timer; /* outer */ -static unsigned long long *list_complete_counter; -static MPIR_T_pvar_timer_t *list_block_timer; /* Inner; while waiting */ /* * These routines provide a default implementation of the MPI RMA operations @@ -1046,7 +504,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) OPA_read_write_barrier(); } - MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_rs); /* This is the second or later fence. Do all the preceding RMA ops. */ comm_ptr = win_ptr->comm_ptr; /* First inform every process whether it is a target of RMA @@ -1096,7 +553,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) mpi_errno = MPIR_Reduce_scatter_block_impl(MPI_IN_PLACE, rma_target_proc, 1, MPI_INT, MPI_SUM, comm_ptr, &errflag); - MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_rs); /* result is stored in rma_target_proc[0] */ if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -1114,15 +570,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) win_ptr->at_completion_counter -= comm_size; win_ptr->at_completion_counter += rma_target_proc[0]; - MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_issue); - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_issue_aux, total_op_count); - MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(RMA, rma_winfence_issue_aux, total_op_count); - - MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(RMA, rma_winfence_reqs, - MPIR_T_PVAR_LEVEL_GET(rma_winfence_reqs)); - MPIR_T_PVAR_LEVEL_SET(RMA, rma_winfence_reqs, 0); /* reset the level */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_reqs, 1); - i = 0; curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); while (curr_ptr != NULL) { @@ -1148,7 +595,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) } else { nRequest++; - MPIR_T_PVAR_LEVEL_INC(RMA, rma_winfence_reqs, 1); curr_ptr = curr_ptr->next; /* The test on the difference is to reduce the number * of times the partial complete routine is called. Without @@ -1161,10 +607,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - MPIR_T_PVAR_STMT(RMA, list_complete_timer = - MPIR_T_PVAR_TIMER_ADDR(rma_winfence_complete)); - MPIR_T_PVAR_STMT(RMA, list_complete_counter = - MPIR_T_PVAR_COUNTER_ADDR(rma_winfence_complete_aux)); mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone); if (mpi_errno != MPI_SUCCESS) @@ -1175,7 +617,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) } } } - MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_issue); /* We replaced a loop over an array of requests with a list of the * incomplete requests. The reason to do @@ -1189,11 +630,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) * needed while looping through the requests. */ if (total_op_count) { - MPIR_T_PVAR_STMT(RMA, list_complete_timer = - MPIR_T_PVAR_TIMER_ADDR(rma_winfence_complete)); - MPIR_T_PVAR_STMT(RMA, list_complete_counter = - MPIR_T_PVAR_COUNTER_ADDR(rma_winfence_complete_aux)); - MPIR_T_PVAR_STMT(RMA, list_block_timer = MPIR_T_PVAR_TIMER_ADDR(rma_winfence_block)); mpi_errno = rma_list_complete(win_ptr, ops_list); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1209,7 +645,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) finish_up: /* wait for all operations from other processes to finish */ if (win_ptr->at_completion_counter) { - MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_wait); MPID_Progress_start(&progress_state); while (win_ptr->at_completion_counter) { mpi_errno = MPID_Progress_wait(&progress_state); @@ -1219,10 +654,8 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**winnoprogress"); } /* --END ERROR HANDLING-- */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_wait_aux, 1); } MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_wait); } if (assert & MPI_MODE_NOSUCCEED) { @@ -2094,8 +1527,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr) MPI_Request *req; MPI_Status *status; - MPIR_T_PVAR_TIMER_START(RMA, rma_winpost_sendsync); - /* NOCHECK not specified. We need to notify the source * processes that Post has been called. */ @@ -2132,7 +1563,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr) "status"); /* Send a 0-byte message to the source processes */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winpost_sendsync_aux, post_grp_size); for (i = 0; i < post_grp_size; i++) { dst = ranks_in_win_grp[i]; @@ -2170,7 +1600,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr) mpi_errno = MPIR_Group_free_impl(win_grp_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); - MPIR_T_PVAR_TIMER_END(RMA, rma_winpost_sendsync); } fn_exit: @@ -2412,8 +1841,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) start_grp_size = win_ptr->start_group_ptr->size; - MPIR_T_PVAR_TIMER_START(RMA, rma_wincomplete_recvsync); - MPIU_CHKLMEM_MALLOC(ranks_in_win_grp, int *, start_grp_size * sizeof(int), mpi_errno, "ranks_in_win_grp"); @@ -2432,15 +1859,12 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) if (mpi_errno) MPIU_ERR_POP(mpi_errno); } - MPIR_T_PVAR_TIMER_END(RMA, rma_wincomplete_recvsync); /* keep track of no. of ops to each proc. Needed for knowing * whether or not to decrement the completion counter. The * completion counter is decremented only on the last * operation. */ - MPIR_T_PVAR_TIMER_START(RMA, rma_wincomplete_issue); - /* Note, active target uses the following ops list, and passive * target uses win_ptr->targets[..] */ ops_list = &win_ptr->at_rma_ops_list; @@ -2457,9 +1881,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) curr_ptr = curr_ptr->next; } - MPIR_T_PVAR_COUNTER_INC(RMA, rma_wincomplete_issue_aux, total_op_count); - MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(RMA, rma_wincomplete_issue_aux, total_op_count); - /* We allocate a few extra requests because if there are no RMA * ops to a target process, we need to send a 0-byte message just * to decrement the completion counter. */ @@ -2493,7 +1914,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) } else { nRequest++; - MPIR_T_PVAR_COUNTER_INC(RMA, rma_wincomplete_reqs, 1); curr_ptr = curr_ptr->next; if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD && nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) { @@ -2501,10 +1921,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - MPIR_T_PVAR_STMT(RMA, list_complete_timer = - MPIR_T_PVAR_TIMER_ADDR(rma_wincomplete_complete)); - MPIR_T_PVAR_STMT(RMA, list_complete_counter = - MPIR_T_PVAR_COUNTER_ADDR(rma_wincomplete_complete_aux)); mpi_errno = rma_list_gc(win_ptr, ops_list, curr_ptr, &nDone); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -2513,7 +1929,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) } } } - MPIR_T_PVAR_TIMER_END(RMA, rma_wincomplete_issue); /* If the start_group included some processes that did not end up * becoming targets of RMA operations from this process, we need @@ -2555,16 +1970,12 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) if (request) { MPIDI_RMA_Op_t *new_ptr = NULL; - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); new_ptr->request = request; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); } j++; new_total_op_count++; @@ -2572,11 +1983,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) } if (new_total_op_count) { - MPIR_T_PVAR_STMT(RMA, list_complete_timer = - MPIR_T_PVAR_TIMER_ADDR(rma_wincomplete_complete)); - MPIR_T_PVAR_STMT(RMA, list_complete_counter = - MPIR_T_PVAR_COUNTER_ADDR(rma_wincomplete_complete_aux)); - MPIR_T_PVAR_STMT(RMA, list_block_timer = MPIR_T_PVAR_TIMER_ADDR(rma_wincomplete_block)); mpi_errno = rma_list_complete(win_ptr, ops_list); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -2626,7 +2032,6 @@ int MPIDI_Win_wait(MPID_Win * win_ptr) if (win_ptr->at_completion_counter) { MPID_Progress_state progress_state; - MPIR_T_PVAR_TIMER_START(RMA, rma_winwait_wait); MPID_Progress_start(&progress_state); while (win_ptr->at_completion_counter) { mpi_errno = MPID_Progress_wait(&progress_state); @@ -2637,10 +2042,8 @@ int MPIDI_Win_wait(MPID_Win * win_ptr) return mpi_errno; } /* --END ERROR HANDLING-- */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winwait_wait_aux, 1); } MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winwait_wait); } /* Ensure ordering of load/store operations. */ @@ -3476,8 +2879,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, curr_ptr = curr_ptr->next; } - MPIR_T_PVAR_TIMER_START(RMA, rma_winunlock_issue); - curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list); if (curr_ptr != NULL) { @@ -3543,7 +2944,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, } else { nRequest++; - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winunlock_reqs, 1); curr_ptr = curr_ptr->next; if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD && nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) { @@ -3551,10 +2951,6 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - MPIR_T_PVAR_STMT(RMA, list_complete_timer = - MPIR_T_PVAR_TIMER_ADDR(rma_winunlock_complete)); - MPIR_T_PVAR_STMT(RMA, list_complete_counter = - MPIR_T_PVAR_COUNTER_ADDR(rma_winunlock_complete_aux)); mpi_errno = rma_list_gc(win_ptr, &win_ptr->targets[target_rank].rma_ops_list, curr_ptr, &nDone); @@ -3566,13 +2962,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, } } } - MPIR_T_PVAR_TIMER_END(RMA, rma_winunlock_issue); if (nops) { - MPIR_T_PVAR_STMT(RMA, list_complete_timer = MPIR_T_PVAR_TIMER_ADDR(rma_winunlock_complete)); - MPIR_T_PVAR_STMT(RMA, list_block_timer = MPIR_T_PVAR_TIMER_ADDR(rma_winunlock_block)); - MPIR_T_PVAR_STMT(RMA, list_complete_counter = - MPIR_T_PVAR_COUNTER_ADDR(rma_winunlock_complete_aux)); mpi_errno = rma_list_complete(win_ptr, &win_ptr->targets[target_rank].rma_ops_list); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -3666,7 +3057,6 @@ static int acquire_local_lock(MPID_Win * win_ptr, int lock_type) if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { MPID_Progress_state progress_state; - MPIR_T_PVAR_TIMER_START(RMA, rma_winlock_getlocallock); MPID_Progress_start(&progress_state); while (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { mpi_errno = MPID_Progress_wait(&progress_state); @@ -3678,7 +3068,6 @@ static int acquire_local_lock(MPID_Win * win_ptr, int lock_type) /* --END ERROR HANDLING-- */ } MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winlock_getlocallock); } win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; @@ -3716,7 +3105,6 @@ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank) if (win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_GRANTED) { MPID_Progress_state progress_state; - MPIR_T_PVAR_TIMER_START(RMA, rma_winunlock_getlock); MPID_Progress_start(&progress_state); while (win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_GRANTED) { mpi_errno = MPID_Progress_wait(&progress_state); @@ -3728,7 +3116,6 @@ static int wait_for_lock_granted(MPID_Win * win_ptr, int target_rank) /* --END ERROR HANDLING-- */ } MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winunlock_getlock); } fn_exit: @@ -4497,7 +3884,6 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt"); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc); MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags); @@ -4524,7 +3910,6 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) { - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc_predef); MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP); req->dev.datatype = accum_pkt->datatype; @@ -4563,11 +3948,9 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_ERR_POP(mpi_errno); if (complete) { *rreqp = NULL; - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_predef); goto fn_exit; } } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_predef); } else { MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT); @@ -4623,7 +4006,6 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } fn_exit: - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE); return mpi_errno; fn_fail: @@ -4649,7 +4031,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate immedidate pkt"); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc_immed); MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, accum_pkt->flags); @@ -4661,7 +4042,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc_immed_op); if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); /* Data is already present */ @@ -4686,7 +4066,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_immed_op); /* There are additional steps to take if this is a passive * target RMA or the last operation from the source */ @@ -4702,7 +4081,6 @@ int MPIDI_CH3_PktHandler_Accumulate_Immed(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt } fn_exit: - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc_immed); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE_IMMED); return mpi_errno; fn_fail: @@ -4731,7 +4109,6 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt"); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_cas); MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL); MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr); mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, cas_pkt->flags); @@ -4797,7 +4174,6 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } fn_exit: - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_cas); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS); return mpi_errno; fn_fail: @@ -4861,7 +4237,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt"); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_fop); MPIU_Assert(fop_pkt->target_win_handle != MPI_WIN_NULL); MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr); mpi_errno = MPIDI_CH3_Start_rma_op_target(win_ptr, fop_pkt->flags); @@ -4925,7 +4300,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, fn_exit: MPIU_CHKPMEM_COMMIT(); - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ @@ -5015,7 +4389,6 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt"); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_accum); data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); @@ -5037,7 +4410,6 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); fn_exit: - MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_accum); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP); return mpi_errno; fn_fail: @@ -5087,9 +4459,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, curr_ptr = curr_ptr->next; } - MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc); new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc); if (!new_ptr) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDI_Win_lock_queue"); @@ -5161,9 +4531,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* queue the information */ MPIDI_Win_lock_queue *curr_ptr, *prev_ptr, *new_ptr; - MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc); new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc); if (!new_ptr) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDI_Win_lock_queue"); @@ -5320,9 +4688,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, curr_ptr = curr_ptr->next; } - MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc); new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc); if (!new_ptr) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDI_Win_lock_queue"); @@ -5401,9 +4767,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* queue the information */ - MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc); new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue)); - MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc); if (!new_ptr) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDI_Win_lock_queue"); @@ -5708,7 +5072,6 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o MPIDI_RMA_Op_t *curr_ptr; MPID_Progress_state progress_state; - MPIR_T_PVAR_TIMER_START_VAR(RMA, list_complete_timer); MPID_Progress_start(&progress_state); /* Process all operations until they are complete */ while (!MPIDI_CH3I_RMA_Ops_isempty(ops_list)) { @@ -5729,7 +5092,6 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o break; } if (curr_ptr && !MPID_Request_is_complete(curr_ptr->request)) { - MPIR_T_PVAR_TIMER_START_VAR(RMA, list_block_timer); mpi_errno = MPID_Progress_wait(&progress_state); /* --BEGIN ERROR HANDLING-- */ if (mpi_errno != MPI_SUCCESS) { @@ -5737,12 +5099,9 @@ static inline int rma_list_complete(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * o MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); } /* --END ERROR HANDLING-- */ - MPIR_T_PVAR_TIMER_END_VAR(RMA, list_block_timer); } } /* While list of rma operation is non-empty */ MPID_Progress_end(&progress_state); - MPIR_T_PVAR_COUNTER_INC_VAR(RMA, list_complete_counter, ntimes); - MPIR_T_PVAR_TIMER_END_VAR(RMA, list_complete_timer); fn_fail: return mpi_errno; @@ -5763,8 +5122,6 @@ static inline int rma_list_gc(MPID_Win * win_ptr, int nComplete = 0; int nVisit = 0; - MPIR_T_PVAR_TIMER_START_VAR(RMA, list_complete_timer); - curr_ptr = MPIDI_CH3I_RMA_Ops_head(ops_list); do { /* MT: avoid processing unissued operations enqueued by other threads @@ -5824,8 +5181,6 @@ static inline int rma_list_gc(MPID_Win * win_ptr, } while (curr_ptr && curr_ptr != last_elm); /* if (nComplete) printf("Completed %d requests\n", nComplete); */ - MPIR_T_PVAR_COUNTER_INC_VAR(RMA, list_complete_counter, 1); - MPIR_T_PVAR_TIMER_END_VAR(RMA, list_complete_timer); *nDone = nComplete; diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index 7beaafe..eb9398b 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -8,8 +8,6 @@ #include "mpiinfo.h" #include "mpidrma.h" -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather); - #undef FUNCNAME #define FUNCNAME MPIDI_Win_fns_init #undef FCNAME @@ -54,7 +52,6 @@ int MPIDI_CH3U_Win_create_gather( void *base, MPI_Aint size, int disp_unit, /* RMA handlers should be set before calling this function */ mpi_errno = (*win_ptr)->RMAFns.Win_set_info(*win_ptr, info); - MPIR_T_PVAR_TIMER_START(RMA, rma_wincreate_allgather); /* allocate memory for the base addresses, disp_units, and completion counters of all processes */ MPIU_CHKPMEM_MALLOC((*win_ptr)->base_addrs, void **, @@ -87,7 +84,6 @@ int MPIDI_CH3U_Win_create_gather( void *base, MPI_Aint size, int disp_unit, mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, tmp_buf, 4, MPI_AINT, (*win_ptr)->comm_ptr, &errflag); - MPIR_T_PVAR_TIMER_END(RMA, rma_wincreate_allgather); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index 428d7a4..139dd76 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -10,12 +10,6 @@ MPIU_THREADSAFE_INIT_DECL(initRMAoptions); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_rs); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winfree_complete); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); -MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set); - static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, MPID_Comm * comm_ptr, MPID_Win ** win_ptr); ----------------------------------------------------------------------- Summary of changes: src/mpi/errhan/errnames.txt | 3 + src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c | 4 - src/mpid/ch3/include/mpid_rma_oplist.h | 270 ++ src/mpid/ch3/include/{mpidrma.h => mpid_rma_shm.h} | 315 +-- src/mpid/ch3/include/mpid_rma_types.h | 102 + src/mpid/ch3/include/mpidimpl.h | 4 +- src/mpid/ch3/include/mpidpkt.h | 51 +- src/mpid/ch3/include/mpidpre.h | 9 + src/mpid/ch3/include/mpidrma.h | 1181 ++----- src/mpid/ch3/src/Makefile.mk | 4 +- src/mpid/ch3/src/ch3u_handle_recv_pkt.c | 4 +- src/mpid/ch3/src/ch3u_handle_recv_req.c | 172 +- src/mpid/ch3/src/ch3u_rma_acc_ops.c | 15 - src/mpid/ch3/src/ch3u_rma_oplist.c | 822 +++++ src/mpid/ch3/src/ch3u_rma_ops.c | 543 ++- src/mpid/ch3/src/ch3u_rma_pkthandler.c | 1757 ++++++++++ src/mpid/ch3/src/ch3u_rma_sync.c | 3699 +------------------- src/mpid/ch3/src/ch3u_win_fns.c | 4 - src/mpid/ch3/src/mpid_finalize.c | 2 + src/mpid/ch3/src/mpid_init.c | 3 + src/mpid/ch3/src/mpid_rma.c | 20 +- src/mpid/ch3/src/mpidi_printf.c | 8 +- src/mpid/ch3/src/mpidi_rma.c | 210 ++ 23 files changed, 3986 insertions(+), 5216 deletions(-) create mode 100644 src/mpid/ch3/include/mpid_rma_oplist.h copy src/mpid/ch3/include/{mpidrma.h => mpid_rma_shm.h} (72%) create mode 100644 src/mpid/ch3/include/mpid_rma_types.h create mode 100644 src/mpid/ch3/src/ch3u_rma_oplist.c create mode 100644 src/mpid/ch3/src/ch3u_rma_pkthandler.c create mode 100644 src/mpid/ch3/src/mpidi_rma.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.3-71-g0086b7b
by noreply＠mpich.org 03 Nov '14

03 Nov '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 0086b7bf7f2ec4e4ef39bca1c153c1000e137a3c (commit) via 017668158a62efeb696c0af1efad8a0ddd8f64b7 (commit) from 72a1e6f871e16943e364dd580e10b392d1e904a4 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/0086b7bf7f2ec4e4ef39bca1c153c1000… commit 0086b7bf7f2ec4e4ef39bca1c153c1000e137a3c Author: Wesley Bland <wbland(a)anl.gov> Date: Sun Nov 2 16:04:20 2014 -0600 Add CVAR for initial size of RTS queue Rather than having a static value for the initial size of the RTS queue, have a CVAR to define it. Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h b/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h index 69b145d..c5b9111 100644 --- a/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h +++ b/src/mpid/ch3/channels/nemesis/include/mpid_nem_impl.h @@ -55,9 +55,9 @@ typedef struct MPID_nem_pkt_lmt_rts } MPID_nem_pkt_lmt_rts_t; -#define MPID_NEM_LMT_RTS_QUEUE_SIZE 1024 extern int *MPID_nem_lmt_rts_queue; extern int MPID_nem_lmt_rts_queue_last_inserted; +extern int MPID_nem_lmt_rts_queue_size; typedef struct MPID_nem_pkt_lmt_cts { diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c index 8fcd181..5ef65de 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c @@ -6,6 +6,24 @@ #include "mpid_nem_impl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_NEM_LMT_RTS_QUEUE_SIZE + category : CH3 + type : int + default : 1024 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + The initial size of the NEM_LMT_RTS_QUEUE used to track RTS + messages before the LMT setup. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + #define set_request_info(rreq_, pkt_, msg_type_) \ { \ (rreq_)->status.MPI_SOURCE = (pkt_)->match.parts.rank; \ @@ -112,7 +130,7 @@ int MPID_nem_lmt_RndvSend(MPID_Request **sreq_p, const void * buf, int count, * queue. It will print a message to warn the user. This should only * affect FT and not matching so we'll consider this ok for now. */ for (i = MPID_nem_lmt_rts_queue_last_inserted + 1; ; i++) { - if (i == MPID_NEM_LMT_RTS_QUEUE_SIZE) { + if (i == MPID_nem_lmt_rts_queue_size) { i = -1; continue; } @@ -329,7 +347,7 @@ static int pkt_CTS_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, MPIDI_msg_sz_t for (i = MPID_nem_lmt_rts_queue_last_inserted + 1; i != MPID_nem_lmt_rts_queue_last_inserted; i++) { - if (i == MPID_NEM_LMT_RTS_QUEUE_SIZE) { + if (i == MPID_nem_lmt_rts_queue_size) { i = -1; continue; } diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c index 6a1715b..d312f4c 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_shm.c @@ -814,7 +814,7 @@ int MPID_nem_lmt_shm_vc_terminated(MPIDI_VC_t *vc) /* If there is anything in the RTS queue, it needs to be cleared out. */ MPIU_THREAD_CS_ENTER(LMT,); - for (i = 0; i < MPID_NEM_LMT_RTS_QUEUE_SIZE; i++) { + for (i = 0; i < MPID_nem_lmt_rts_queue_size; i++) { if (MPI_REQUEST_NULL != MPID_nem_lmt_rts_queue[i]) { MPID_Request_get_ptr(MPID_nem_lmt_rts_queue[i], req); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Checking RTS message"); diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_mpich.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_mpich.c index 9e7fc5b..ee267bd 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_mpich.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_mpich.c @@ -24,6 +24,7 @@ MPID_nem_cell_ptr_t MPID_nem_prefetched_cell = 0; unsigned short *MPID_nem_recv_seqno = 0; int *MPID_nem_lmt_rts_queue; +int MPID_nem_lmt_rts_queue_size; int MPID_nem_lmt_rts_queue_last_inserted = 0; #undef FUNCNAME @@ -71,8 +72,9 @@ MPID_nem_mpich_init(void) * RTS requests. If we run out of space, we'll just drop the extra * requests. This won't cause a matching problem, it will just prevent FT * from working for those requests that get dropped. */ - MPIU_CHKPMEM_MALLOC(MPID_nem_lmt_rts_queue, int *, sizeof(int) * MPID_NEM_LMT_RTS_QUEUE_SIZE, mpi_errno, "lmt rts queue"); - for (i = 0; i < MPID_NEM_LMT_RTS_QUEUE_SIZE; i++) + MPID_nem_lmt_rts_queue_size = MPIR_CVAR_NEM_LMT_RTS_QUEUE_SIZE; + MPIU_CHKPMEM_MALLOC(MPID_nem_lmt_rts_queue, int *, sizeof(int) * MPID_nem_lmt_rts_queue_size, mpi_errno, "lmt rts queue"); + for (i = 0; i < MPID_nem_lmt_rts_queue_size; i++) MPID_nem_lmt_rts_queue[i] = MPI_REQUEST_NULL; MPIU_CHKPMEM_COMMIT(); http://git.mpich.org/mpich.git/commitdiff/017668158a62efeb696c0af1efad8a0dd… commit 017668158a62efeb696c0af1efad8a0ddd8f64b7 Author: Wesley Bland <wbland(a)anl.gov> Date: Sun Nov 2 16:04:20 2014 -0600 Add a warning if the RTS queue is overflowed When the RTS queue fills up the first time, print out a warning to let the user know that they've done it and FT won't be provided anymore. Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c index f7396c8..8fcd181 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c @@ -28,6 +28,8 @@ static int pkt_CTS_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, MPIDI_msg_sz_t static int pkt_DONE_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, MPIDI_msg_sz_t *buflen, MPID_Request **rreqp); static int pkt_COOKIE_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, MPIDI_msg_sz_t *buflen, MPID_Request **rreqp); +static int warning_printed = 0; + #undef FUNCNAME #define FUNCNAME MPID_nem_lmt_pkthandler_init #undef FCNAME @@ -107,11 +109,9 @@ int MPID_nem_lmt_RndvSend(MPID_Request **sreq_p, const void * buf, int count, if (MPI_SUCCESS == mpi_errno) { /* If this loops all the way around and can't find a place to put the * RTS request, it will just drop the request and leave it out of the - * queue silently. This should only affect FT and not matching so we'll - * consider this ok for now. */ - for (i = MPID_nem_lmt_rts_queue_last_inserted + 1; - i != MPID_nem_lmt_rts_queue_last_inserted; - i++) { + * queue. It will print a message to warn the user. This should only + * affect FT and not matching so we'll consider this ok for now. */ + for (i = MPID_nem_lmt_rts_queue_last_inserted + 1; ; i++) { if (i == MPID_NEM_LMT_RTS_QUEUE_SIZE) { i = -1; continue; @@ -122,6 +122,12 @@ int MPID_nem_lmt_RndvSend(MPID_Request **sreq_p, const void * buf, int count, MPID_nem_lmt_rts_queue_last_inserted = i; break; } + + if (i == MPID_nem_lmt_rts_queue_last_inserted && !warning_printed) { + MPIU_Internal_error_printf("LMT RTS queue exceeded. FT not provided for overflowed messages.\n"); + warning_printed = 1; + break; + } } } MPIU_THREAD_CS_EXIT(LMT,); ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/include/mpid_nem_impl.h | 2 +- src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c | 38 ++++++++++++++++---- .../ch3/channels/nemesis/src/mpid_nem_lmt_shm.c | 2 +- src/mpid/ch3/channels/nemesis/src/mpid_nem_mpich.c | 6 ++- 4 files changed, 37 insertions(+), 11 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.3-69-g72a1e6f
by noreply＠mpich.org 01 Nov '14

01 Nov '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 72a1e6f871e16943e364dd580e10b392d1e904a4 (commit) via 5f78f0fa399747dd23201f38fae773dda9af3f4d (commit) from c76aa786e784faf0302367f7fc46d62f399bcbba (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/72a1e6f871e16943e364dd580e10b392d… commit 72a1e6f871e16943e364dd580e10b392d1e904a4 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Thu Oct 30 15:57:02 2014 -0500 Bug-fix: avoid free NULL pointer in RMA. req->dev.user_buf points to the data sent from origin process to target process, and for FOP sometimes it points to the IMMED area in packet header when data can be fit in packet header. In such case, we should not free req->dev.user_buf in final request handler since that data area will be freed by the runtime when packet header is freed. In this patch we initialize user_buf to NULL when creating the request, and set it to NULL when FOP is completed, and avoid free a NULL pointer in final request handler. Signed-off-by: Min Si <msi(a)il.is.s.u-tokyo.ac.jp> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 8031d04..db6760c 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -322,7 +322,8 @@ int MPIDI_CH3_ReqHandler_GetAccumRespComplete( MPIDI_VC_t *vc, MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GETACCUMRESPCOMPLETE); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GETACCUMRESPCOMPLETE); - MPIU_Free(rreq->dev.user_buf); + if (rreq->dev.user_buf != NULL) + MPIU_Free(rreq->dev.user_buf); MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); @@ -616,6 +617,16 @@ int MPIDI_CH3_ReqHandler_FOPComplete( MPIDI_VC_t *vc, /* Free temporary buffer allocated in PktHandler_FOP */ if (len > sizeof(int) * MPIDI_RMA_FOP_IMMED_INTS && rreq->dev.op != MPI_NO_OP) { MPIU_Free(rreq->dev.user_buf); + /* Assign user_buf to NULL so that reqHandler_GetAccumRespComplete() + will not try to free an empty buffer. */ + rreq->dev.user_buf = NULL; + } + else { + /* FOP data fit in pkt header and user_buf just points to data area in pkt header + in pktHandler_FOP(), and it should be freed when pkt header is freed. + Here we assign user_buf to NULL so that reqHandler_GetAccumRespComplete() + will not try to free it. */ + rreq->dev.user_buf = NULL; } *complete = 1; diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c index c5ff48d..7caa636 100644 --- a/src/mpid/ch3/src/ch3u_request.c +++ b/src/mpid/ch3/src/ch3u_request.c @@ -88,6 +88,7 @@ MPID_Request * MPID_Request_create(void) req->dev.iov_offset = 0; req->dev.flags = MPIDI_CH3_PKT_FLAG_NONE; req->dev.resp_request_handle = MPI_REQUEST_NULL; + req->dev.user_buf = NULL; req->dev.OnDataAvail = NULL; req->dev.OnFinal = NULL; #ifdef MPIDI_CH3_REQUEST_INIT http://git.mpich.org/mpich.git/commitdiff/5f78f0fa399747dd23201f38fae773dda… commit 5f78f0fa399747dd23201f38fae773dda9af3f4d Author: Igor Ivanov <Igor.Ivanov(a)itseez.com> Date: Mon Oct 20 17:35:05 2014 +0200 netmod/mxm: Avoid calling mxm send req handling from mxm send completion callback Signed-off-by: Devendar Bureddy <devendar(a)mellanox.com> Signed-off-by: Igor Ivanov <Igor.Ivanov(a)itseez.com> diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h index 3b2bb12..43070da 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h @@ -69,6 +69,8 @@ void MPID_nem_mxm_get_adi_msg(mxm_conn_h conn, mxm_imm_t imm, void *data, void MPID_nem_mxm_anysource_posted(MPID_Request * req); int MPID_nem_mxm_anysource_matched(MPID_Request * req); +int _mxm_handle_sreq(MPID_Request * req); + /* List type as queue * Operations, initialization etc */ @@ -174,6 +176,25 @@ typedef struct { /* macro for mxm private in REQ */ #define REQ_BASE(reqp) ((reqp) ? (MPID_nem_mxm_req_area *)((reqp)->ch.netmod_area.padding) : NULL) +typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_mxm_reqq_t; +#define MPID_nem_mxm_queue_empty(q) GENERIC_Q_EMPTY (q) +#define MPID_nem_mxm_queue_head(q) GENERIC_Q_HEAD (q) +#define MPID_nem_mxm_queue_enqueue(qp, ep) do { \ + /* add refcount so req doesn't get freed before it's dequeued */ \ + MPIR_Request_add_ref(ep); \ + MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, \ + "MPID_nem_mxm_queue_enqueue req=%p (handle=%#x), queue=%p", \ + ep, (ep)->handle, qp)); \ + GENERIC_Q_ENQUEUE (qp, ep, dev.next); \ + } while (0) +#define MPID_nem_mxm_queue_dequeue(qp, ep) do { \ + GENERIC_Q_DEQUEUE (qp, ep, dev.next); \ + MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, \ + "MPID_nem_mxm_queue_dequeuereq=%p (handle=%#x), queue=%p", \ + *(ep), *(ep) ? (*(ep))->handle : -1, qp)); \ + MPID_Request_release(*(ep)); \ + } while (0) + typedef struct MPID_nem_mxm_module_t { char *runtime_version; const char *compiletime_version; @@ -188,6 +209,7 @@ typedef struct MPID_nem_mxm_module_t { int mxm_np; MPID_nem_mxm_ep_t *endpoint; list_head_t free_queue; + MPID_nem_mxm_reqq_t sreq_queue; struct { int bulk_connect; /* use bulk connect */ int bulk_disconnect; /* use bulk disconnect */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c index 949efc7..4d78565 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c @@ -482,6 +482,8 @@ static int _mxm_init(int rank, int size) list_grow_mxm_req(&_mxm_obj.free_queue); MPIU_Assert(list_length(&_mxm_obj.free_queue) == MXM_MPICH_MAX_REQ); + _mxm_obj.sreq_queue.head = _mxm_obj.sreq_queue.tail = NULL; + mxm_obj = &_mxm_obj; fn_exit: diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c index ba7686e..e8bddc3 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c @@ -24,10 +24,16 @@ static int _mxm_process_rdtype(MPID_Request ** rreq_p, MPI_Datatype datatype, int MPID_nem_mxm_poll(int in_blocking_progress) { int mpi_errno = MPI_SUCCESS; + MPID_Request *req = NULL; MPIDI_STATE_DECL(MPID_STATE_MXM_POLL); MPIDI_FUNC_ENTER(MPID_STATE_MXM_POLL); + while (!MPID_nem_mxm_queue_empty(mxm_obj->sreq_queue)) { + MPID_nem_mxm_queue_dequeue(&mxm_obj->sreq_queue, &req); + _mxm_handle_sreq(req); + } + mpi_errno = _mxm_poll(); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -72,6 +78,7 @@ void MPID_nem_mxm_get_adi_msg(mxm_conn_h conn, mxm_imm_t imm, void *data, vc = mxm_conn_ctx_get(conn); _dbg_mxm_output(5, "========> Getting ADI msg (from=%d data_size %d) \n", vc->pg_rank, length); + _dbg_mxm_out_buf(data, (length > 16 ? 16 : length)); MPID_nem_handle_pkt(vc, data, (MPIDI_msg_sz_t) (length)); } @@ -144,6 +151,10 @@ int MPID_nem_mxm_anysource_matched(MPID_Request * req) int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) { int mpi_errno = MPI_SUCCESS; + MPIDI_msg_sz_t data_sz; + int dt_contig; + MPI_Aint dt_true_lb; + MPID_Datatype *dt_ptr; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_RECV); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_RECV); @@ -152,18 +163,15 @@ int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) MPIU_Assert(((rreq->dev.match.parts.rank == MPI_ANY_SOURCE) && (vc == NULL)) || (vc && !vc->ch.is_local)); + MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz, + dt_ptr, dt_true_lb); + { MPIR_Context_id_t context_id = rreq->dev.match.parts.context_id; int tag = rreq->dev.match.parts.tag; - MPIDI_msg_sz_t data_sz; - int dt_contig; - MPI_Aint dt_true_lb; - MPID_Datatype *dt_ptr; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; - MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz, - dt_ptr, dt_true_lb); rreq->dev.OnDataAvail = NULL; rreq->dev.tmpbuf = NULL; rreq->ch.vc = vc; @@ -223,7 +231,6 @@ static int _mxm_handle_rreq(MPID_Request * req) MPIDI_msg_sz_t userbuf_sz; MPID_Datatype *dt_ptr; MPIDI_msg_sz_t data_sz; - MPIDI_VC_t *vc = NULL; MPID_nem_mxm_vc_area *vc_area ATTRIBUTE((unused)) = NULL; MPID_nem_mxm_req_area *req_area = NULL; void *tmp_buf = NULL; @@ -319,7 +326,7 @@ static int _mxm_handle_rreq(MPID_Request * req) } } - MPIDI_CH3U_Handle_recv_req(vc, req, &complete); + MPIDI_CH3U_Handle_recv_req(req->ch.vc, req, &complete); MPIU_Assert(complete == TRUE); if (tmp_buf) MPIU_Free(tmp_buf); diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c index 75003bf..69f3adc 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c @@ -15,7 +15,6 @@ enum { }; -static int _mxm_handle_sreq(MPID_Request * req); static void _mxm_send_completion_cb(void *context); static int _mxm_isend(MPID_nem_mxm_ep_t * ep, MPID_nem_mxm_req_area * req, int type, mxm_mq_h mxm_mq, int mxm_rank, int id, mxm_tag_t tag, int block); @@ -235,6 +234,7 @@ int MPID_nem_mxm_send(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit); MPIU_Assert(sreq != NULL); MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND); + MPIDI_VC_FAI_send_seqnum(vc, seqnum); MPIDI_Request_set_seqnum(sreq, seqnum); if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { @@ -336,7 +336,8 @@ int MPID_nem_mxm_ssend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype /* create a request */ MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit); MPIU_Assert(sreq != NULL); - MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND); + MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SSEND); + MPIDI_VC_FAI_send_seqnum(vc, seqnum); MPIDI_Request_set_seqnum(sreq, seqnum); if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { @@ -439,6 +440,7 @@ int MPID_nem_mxm_isend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit); MPIU_Assert(sreq != NULL); MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND); + MPIDI_VC_FAI_send_seqnum(vc, seqnum); MPIDI_Request_set_seqnum(sreq, seqnum); if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { @@ -541,7 +543,8 @@ int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatyp /* create a request */ MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit); MPIU_Assert(sreq != NULL); - MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND); + MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SSEND); + MPIDI_VC_FAI_send_seqnum(vc, seqnum); MPIDI_Request_set_seqnum(sreq, seqnum); if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { @@ -619,10 +622,9 @@ int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatyp } -static int _mxm_handle_sreq(MPID_Request * req) +int _mxm_handle_sreq(MPID_Request * req) { int complete = FALSE; - int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; @@ -634,8 +636,10 @@ static int _mxm_handle_sreq(MPID_Request * req) 16 ? 16 : req_area->iov_buf[0].length)); vc_area->pending_sends -= 1; - if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))) { - MPIU_Free(req->dev.tmpbuf); + if (req->dev.tmpbuf) { + if (req->dev.datatype_ptr || req->ch.noncontig) { + MPIU_Free(req->dev.tmpbuf); + } } if (req_area->iov_count > MXM_MPICH_MAX_IOV) { @@ -644,19 +648,8 @@ static int _mxm_handle_sreq(MPID_Request * req) req_area->iov_count = 0; } - reqFn = req->dev.OnDataAvail; - if (!reqFn) { - MPIDI_CH3U_Request_complete(req); - MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); - } - else { - MPIDI_VC_t *vc = req->ch.vc; - - reqFn(vc, req, &complete); - if (!complete) { - MPIU_Assert(complete == TRUE); - } - } + MPIDI_CH3U_Handle_send_req(req->ch.vc, req, &complete); + MPIU_Assert(complete == TRUE); return complete; } @@ -683,7 +676,7 @@ static void _mxm_send_completion_cb(void *context) req, req->status.MPI_ERROR); if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) { - _mxm_handle_sreq(req); + MPID_nem_mxm_queue_enqueue(&mxm_obj->sreq_queue, req); } } ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/netmod/mxm/mxm_impl.h | 22 ++++++++++++ .../ch3/channels/nemesis/netmod/mxm/mxm_init.c | 2 + .../ch3/channels/nemesis/netmod/mxm/mxm_poll.c | 23 ++++++++---- .../ch3/channels/nemesis/netmod/mxm/mxm_send.c | 35 ++++++++------------ src/mpid/ch3/src/ch3u_handle_recv_req.c | 13 +++++++- src/mpid/ch3/src/ch3u_request.c | 1 + 6 files changed, 66 insertions(+), 30 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.3-67-gc76aa78
by noreply＠mpich.org 01 Nov '14

01 Nov '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via c76aa786e784faf0302367f7fc46d62f399bcbba (commit) from 9164deed691140ca0f136e2ac6297248c8f3564f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/c76aa786e784faf0302367f7fc46d62f3… commit c76aa786e784faf0302367f7fc46d62f399bcbba Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Aug 12 15:44:05 2014 -0500 Bug-fix: always waiting for remote completion in Win_unlock. The original implementation includes an optimization which allows Win_unlock for exclusive lock to return without waiting for remote completion. This relys on the assumption that window memory on target process will not be accessed by a third party until that target process finishes all RMA operations and grants the lock to other processes. However, this assumption is not correct if user uses assert MPI_MODE_NOCHECK. Consider the following code: P0 P1 P2 MPI_Win_lock(P1, NULL, exclusive); MPI_Put(X); MPI_Win_unlock(P1, exclusive); MPI_Send (P2); MPI_Recv(P0); MPI_Win_lock(P1, MODE_NOCHECK, exclusive); MPI_Get(X); MPI_Win_unlock(P1, exclusive); Both P0 and P2 issue exclusive lock to P1, and P2 uses assert MPI_MODE_NOCHECK because the lock should be granted to P2 after synchronization between P2 and P0. However, in the original implementation, GET operation on P2 might not get the updated value since Win_unlock on P0 return without waiting for remote completion. In this patch we delete this optimization. In Win_free, since every Win_unlock guarantees the remote completion, target process no longer needs to do additional counting works to detect target-side completion, but only needs to do a global barrier. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c index 2bc6951..bff1b52 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c @@ -63,6 +63,7 @@ fn_fail: int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) { int mpi_errno = MPI_SUCCESS; + int errflag = FALSE; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_SHM_WIN_FREE); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3_SHM_WIN_FREE); @@ -72,8 +73,9 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) goto fn_exit; } - mpi_errno = MPIDI_CH3I_Wait_for_pt_ops_finish(*win_ptr); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); /* Free shared memory region */ if ((*win_ptr)->shm_allocated) { diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index eebca52..c786dcb 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -338,11 +338,6 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * comm_size*sizeof(MPI_Win), mpi_errno, "(*win_ptr)->all_win_handles"); - MPIU_CHKPMEM_MALLOC((*win_ptr)->pt_rma_puts_accs, int *, - comm_size*sizeof(int), - mpi_errno, "(*win_ptr)->pt_rma_puts_accs"); - for (i=0; i<comm_size; i++) (*win_ptr)->pt_rma_puts_accs[i] = 0; - /* get the sizes of the windows and window objectsof all processes. allocate temp. buffer for communication */ MPIU_CHKLMEM_MALLOC(tmp_buf, MPI_Aint *, 3*comm_size*sizeof(MPI_Aint), mpi_errno, "tmp_buf"); diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 4cf9d32..3d59636 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -277,12 +277,6 @@ struct MPIDI_Win_target_state { volatile int shared_lock_ref_cnt; \ struct MPIDI_Win_lock_queue volatile *lock_queue; /* list of unsatisfied locks */ \ \ - int *pt_rma_puts_accs; /* array containing the no. of passive target\ - puts/accums issued from this process to other \ - processes. */ \ - volatile int my_pt_rma_puts_accs; /* no. of passive target puts/accums \ - that this process has \ - completed as target */ \ MPI_Aint *sizes; /* array of sizes of all windows */ \ struct MPIDI_Win_info_args info_args; \ struct MPIDI_Win_target_state *targets; /* Target state and ops \ diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 3048056..652af02 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -1065,57 +1065,6 @@ static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_ad #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_Wait_for_pt_ops_finish -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_Wait_for_pt_ops_finish(MPID_Win * win_ptr) -{ - int mpi_errno = MPI_SUCCESS, total_pt_rma_puts_accs; - MPID_Comm *comm_ptr; - int errflag = FALSE; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_WAIT_FOR_PT_OPS_FINISH); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_WAIT_FOR_PT_OPS_FINISH); - - comm_ptr = win_ptr->comm_ptr; - MPIR_T_PVAR_TIMER_START(RMA, rma_winfree_rs); - mpi_errno = MPIR_Reduce_scatter_block_impl(win_ptr->pt_rma_puts_accs, - &total_pt_rma_puts_accs, 1, - MPI_INT, MPI_SUM, comm_ptr, &errflag); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); - MPIR_T_PVAR_TIMER_END(RMA, rma_winfree_rs); - - if (total_pt_rma_puts_accs != win_ptr->my_pt_rma_puts_accs) { - MPID_Progress_state progress_state; - - /* poke the progress engine until the two are equal */ - MPIR_T_PVAR_TIMER_START(RMA, rma_winfree_complete); - MPID_Progress_start(&progress_state); - while (total_pt_rma_puts_accs != win_ptr->my_pt_rma_puts_accs) { - mpi_errno = MPID_Progress_wait(&progress_state); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPID_Progress_end(&progress_state); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); - } - /* --END ERROR HANDLING-- */ - } - MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winfree_complete); - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_WAIT_FOR_PT_OPS_FINISH); - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME #undef FCNAME #endif diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 262cf71..8031d04 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -1037,9 +1037,6 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) MPIDI_CH3_Finish_rma_op_target(). That call cannot be used here, because it would enter this function recursively. */ - /* increment counter */ - win_ptr->my_pt_rma_puts_accs++; - mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(lock_queue->vc, win_ptr, lock_queue->source_win_handle); diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 3d4e273..83b1e37 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -40,6 +40,7 @@ int MPIDI_Win_free(MPID_Win ** win_ptr) int mpi_errno = MPI_SUCCESS; int in_use; MPID_Comm *comm_ptr; + int errflag = FALSE; MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_FREE); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_FREE); @@ -47,9 +48,13 @@ int MPIDI_Win_free(MPID_Win ** win_ptr) MPIU_ERR_CHKANDJUMP((*win_ptr)->epoch_state != MPIDI_EPOCH_NONE, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - mpi_errno = MPIDI_CH3I_Wait_for_pt_ops_finish(*win_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); + if (!(*win_ptr)->shm_allocated) { + /* when SHM is allocated, we already did a global barrier in + MPIDI_CH3_SHM_Win_free, so we do not need to do it again here. */ + mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + } comm_ptr = (*win_ptr)->comm_ptr; mpi_errno = MPIR_Comm_free_impl(comm_ptr); @@ -61,7 +66,6 @@ int MPIDI_Win_free(MPID_Win ** win_ptr) MPIU_Free((*win_ptr)->sizes); MPIU_Free((*win_ptr)->disp_units); MPIU_Free((*win_ptr)->all_win_handles); - MPIU_Free((*win_ptr)->pt_rma_puts_accs); /* Free the attached buffer for windows created with MPI_Win_allocate() */ if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE || diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index ad42680..416f2f5 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -932,16 +932,12 @@ static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, const void *o_addr, int o_count, MPI_Datatype o_datatype, MPID_Datatype ** combined_dtp); -/* Issue an RMA operation -- Before calling this macro, you must define the - * MPIDI_CH3I_TRACK_RMA_WRITE helper macro. This macro defines any extra action - * that should be taken when a write (put/acc) operation is encountered. */ #define MPIDI_CH3I_ISSUE_RMA_OP(op_ptr_, win_ptr_, flags_, source_win_handle_, target_win_handle_,err_) \ do { \ switch ((op_ptr_)->type) \ { \ case (MPIDI_RMA_PUT): \ case (MPIDI_RMA_ACCUMULATE): \ - MPIDI_CH3I_TRACK_RMA_WRITE(op_ptr_, win_ptr_); \ (err_) = send_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \ (target_win_handle_), &(op_ptr_)->dtype_info, \ &(op_ptr_)->dataloop, &(op_ptr_)->request); \ @@ -962,7 +958,6 @@ static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, (target_win_handle_), &(op_ptr_)->dtype_info, \ &(op_ptr_)->dataloop, &(op_ptr_)->request); \ } else { \ - MPIDI_CH3I_TRACK_RMA_WRITE(op_ptr_, win_ptr_); \ (err_) = send_rma_msg((op_ptr_), (win_ptr_), (flags_), (source_win_handle_), \ (target_win_handle_), &(op_ptr_)->dtype_info, \ &(op_ptr_)->dataloop, &(op_ptr_)->request); \ @@ -970,7 +965,6 @@ static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, if (err_) { MPIU_ERR_POP(err_); } \ break; \ case MPIDI_RMA_ACC_CONTIG: \ - MPIDI_CH3I_TRACK_RMA_WRITE(op_ptr_, win_ptr_); \ (err_) = send_contig_acc_msg((op_ptr_), (win_ptr_), (flags_), \ (source_win_handle_), (target_win_handle_), \ &(op_ptr_)->request); \ @@ -985,7 +979,6 @@ static int create_datatype(const MPIDI_RMA_dtype_info * dtype_info, break; \ case (MPIDI_RMA_COMPARE_AND_SWAP): \ case (MPIDI_RMA_FETCH_AND_OP): \ - MPIDI_CH3I_TRACK_RMA_WRITE(op_ptr_, win_ptr_); \ (err_) = send_immed_rmw_msg((op_ptr_), (win_ptr_), (flags_), \ (source_win_handle_), (target_win_handle_), \ &(op_ptr_)->request); \ @@ -1022,31 +1015,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) win_ptr->epoch_state != MPIDI_EPOCH_FENCE, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - /* In case this process was previously the target of passive target rma - * operations, we need to take care of the following... - * Since we allow MPI_Win_unlock to return without a done ack from - * the target in the case of multiple rma ops and exclusive lock, - * we need to check whether there is a lock on the window, and if - * there is a lock, poke the progress engine until the operartions - * have completed and the lock is released. */ - if (win_ptr->current_lock_type != MPID_LOCK_NONE) { - MPIR_T_PVAR_TIMER_START(RMA, rma_winfence_clearlock); - MPID_Progress_start(&progress_state); - while (win_ptr->current_lock_type != MPID_LOCK_NONE) { - /* poke the progress engine */ - mpi_errno = MPID_Progress_wait(&progress_state); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPID_Progress_end(&progress_state); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); - } - /* --END ERROR HANDLING-- */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winfence_clearlock_aux, 1); - } - MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winfence_clearlock); - } - /* Note that the NOPRECEDE and NOSUCCEED must be specified by all processes * in the window's group if any specify it */ if (assert & MPI_MODE_NOPRECEDE) { @@ -1169,10 +1137,8 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) source_win_handle = win_ptr->handle; target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; -#define MPIDI_CH3I_TRACK_RMA_WRITE(op_ptr_, win_ptr_) /* Not used by active mode */ MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, source_win_handle, target_win_handle, mpi_errno); -#undef MPIDI_CH3I_TRACK_RMA_WRITE i++; curr_ops_cnt[curr_ptr->target_rank]++; @@ -2114,33 +2080,6 @@ int MPIDI_Win_post(MPID_Group * post_grp_ptr, int assert, MPID_Win * win_ptr) * synchronization, we cannot do this because fence_issued must be * updated collectively */ - /* In case this process was previously the target of passive target rma - * operations, we need to take care of the following... - * Since we allow MPI_Win_unlock to return without a done ack from - * the target in the case of multiple rma ops and exclusive lock, - * we need to check whether there is a lock on the window, and if - * there is a lock, poke the progress engine until the operations - * have completed and the lock is therefore released. */ - if (win_ptr->current_lock_type != MPID_LOCK_NONE) { - MPID_Progress_state progress_state; - - MPIR_T_PVAR_TIMER_START(RMA, rma_winpost_clearlock); - /* poke the progress engine */ - MPID_Progress_start(&progress_state); - while (win_ptr->current_lock_type != MPID_LOCK_NONE) { - mpi_errno = MPID_Progress_wait(&progress_state); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPID_Progress_end(&progress_state); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); - } - /* --END ERROR HANDLING-- */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winpost_clearlock_aux, 1); - } - MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winpost_clearlock); - } - post_grp_size = post_grp_ptr->size; /* Ensure ordering of load/store operations. */ @@ -2395,33 +2334,6 @@ int MPIDI_Win_start(MPID_Group * group_ptr, int assert, MPID_Win * win_ptr) * synchronization, we cannot do this because fence_issued must be * updated collectively */ - /* In case this process was previously the target of passive target rma - * operations, we need to take care of the following... - * Since we allow MPI_Win_unlock to return without a done ack from - * the target in the case of multiple rma ops and exclusive lock, - * we need to check whether there is a lock on the window, and if - * there is a lock, poke the progress engine until the operations - * have completed and the lock is therefore released. */ - if (win_ptr->current_lock_type != MPID_LOCK_NONE) { - MPID_Progress_state progress_state; - - MPIR_T_PVAR_TIMER_START(RMA, rma_winstart_clearlock); - /* poke the progress engine */ - MPID_Progress_start(&progress_state); - while (win_ptr->current_lock_type != MPID_LOCK_NONE) { - mpi_errno = MPID_Progress_wait(&progress_state); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno != MPI_SUCCESS) { - MPID_Progress_end(&progress_state); - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**winnoprogress"); - } - /* --END ERROR HANDLING-- */ - MPIR_T_PVAR_COUNTER_INC(RMA, rma_winstart_clearlock_aux, 1); - } - MPID_Progress_end(&progress_state); - MPIR_T_PVAR_TIMER_END(RMA, rma_winstart_clearlock); - } - win_ptr->start_group_ptr = group_ptr; MPIR_Group_add_ref(group_ptr); win_ptr->start_assert = assert; @@ -2570,10 +2482,8 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) source_win_handle = win_ptr->handle; target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank]; -#define MPIDI_CH3I_TRACK_RMA_WRITE(op_ptr_, win_ptr_) /* Not used by active mode */ MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, source_win_handle, target_win_handle, mpi_errno); -#undef MPIDI_CH3I_TRACK_RMA_WRITE i++; curr_ops_cnt[curr_ptr->target_rank]++; @@ -3512,19 +3422,7 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, (win_ptr->targets[target_rank].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED && win_ptr->targets[target_rank].remote_lock_assert & MPI_MODE_NOCHECK)); - if (win_ptr->targets[target_rank].remote_lock_mode == MPI_LOCK_EXCLUSIVE && - win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_CALLED && - win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_FLUSH) { - /* Exclusive lock already held -- no need to wait for rma done pkt at - * the end. This is because the target won't grant another process - * access to the window until all of our operations complete at that - * target. Thus, there is no third-party communication issue. - * However, flush still needs to wait for rma done, otherwise result - * may be unknown if user reads the updated location from a shared window of - * another target process after this flush. */ - *wait_for_rma_done_pkt = 0; - } - else if (MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->targets[target_rank].rma_ops_list)) { + if (MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->targets[target_rank].rma_ops_list)) { /* The ops list is empty -- NOTE: we assume this is because the epoch * was flushed. Any issued ops are already remote complete; done * packet is not needed for safe third party communication. */ @@ -3635,15 +3533,8 @@ static int do_passive_target_rma(MPID_Win * win_ptr, int target_rank, source_win_handle = win_ptr->handle; } - /* Track passive target write operations. This is used during Win_free - * to ensure that all writes to a given target have completed at that - * process before the window is freed. */ -#define MPIDI_CH3I_TRACK_RMA_WRITE(op_, win_ptr_) \ - do { (win_ptr_)->pt_rma_puts_accs[(op_)->target_rank]++; } while (0) - MPIDI_CH3I_ISSUE_RMA_OP(curr_ptr, win_ptr, flags, source_win_handle, target_win_handle, mpi_errno); -#undef MPIDI_CH3I_TRACK_RMA_WRITE /* If the request is null, we can remove it immediately */ if (!curr_ptr->request) { @@ -3969,8 +3860,6 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank) rma_op = MPIDI_CH3I_RMA_Ops_head(&win_ptr->targets[target_rank].rma_ops_list); - win_ptr->pt_rma_puts_accs[rma_op->target_rank]++; - if (rma_op->type == MPIDI_RMA_PUT) { MPIDI_Pkt_init(lock_put_unlock_pkt, MPIDI_CH3_PKT_LOCK_PUT_UNLOCK); lock_put_unlock_pkt->flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK | @@ -6001,12 +5890,6 @@ int MPIDI_CH3_Finish_rma_op_target(MPIDI_VC_t * vc, MPID_Win * win_ptr, int is_r /* This function should be called by the target process after each RMA * operation is completed, to update synchronization state. */ - /* If this is a passive target RMA update operation, increment counter. This is - * needed in Win_free to ensure that all ops are completed before a window - * is freed. */ - if (win_ptr->current_lock_type != MPID_LOCK_NONE && is_rma_update) - win_ptr->my_pt_rma_puts_accs++; - /* Last RMA operation from source. If active target RMA, decrement window * counter. */ if (flags & MPIDI_CH3_PKT_FLAG_RMA_AT_COMPLETE) { diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index 9e16ce8..7beaafe 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -71,11 +71,6 @@ int MPIDI_CH3U_Win_create_gather( void *base, MPI_Aint size, int disp_unit, comm_size*sizeof(MPI_Win), mpi_errno, "(*win_ptr)->all_win_handles"); - MPIU_CHKPMEM_MALLOC((*win_ptr)->pt_rma_puts_accs, int *, - comm_size*sizeof(int), - mpi_errno, "(*win_ptr)->pt_rma_puts_accs"); - for (i=0; i<comm_size; i++) (*win_ptr)->pt_rma_puts_accs[i] = 0; - /* get the addresses of the windows, window objects, and completion counters of all processes. allocate temp. buffer for communication */ MPIU_CHKLMEM_MALLOC(tmp_buf, MPI_Aint *, 4*comm_size*sizeof(MPI_Aint), diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index 639a8c5..428d7a4 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -307,8 +307,6 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->current_lock_type = MPID_LOCK_NONE; (*win_ptr)->shared_lock_ref_cnt = 0; (*win_ptr)->lock_queue = NULL; - (*win_ptr)->pt_rma_puts_accs = NULL; - (*win_ptr)->my_pt_rma_puts_accs = 0; (*win_ptr)->epoch_state = MPIDI_EPOCH_NONE; (*win_ptr)->epoch_count = 0; (*win_ptr)->at_rma_ops_list = NULL; ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c | 6 +- src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c | 5 - src/mpid/ch3/include/mpidpre.h | 6 - src/mpid/ch3/include/mpidrma.h | 51 ---------- src/mpid/ch3/src/ch3u_handle_recv_req.c | 3 - src/mpid/ch3/src/ch3u_rma_ops.c | 12 ++- src/mpid/ch3/src/ch3u_rma_sync.c | 119 +---------------------- src/mpid/ch3/src/ch3u_win_fns.c | 5 - src/mpid/ch3/src/mpid_rma.c | 2 - 9 files changed, 13 insertions(+), 196 deletions(-) hooks/post-receive -- MPICH primary repository

1 0