This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 7d0d237967b9232cf8c2ad30a26d3efd2224e957 (commit) via dbb7afb47e3e6f515f358986acb0a5a0d4944e42 (commit) via e8a3c9d9178f95dc1f803e84982e56cf739d3639 (commit) via b7066037f8b539ef9979984d2ebdbaca0322e1bf (commit) via 67bbc9a50f61aef5bf53fd07c84775f11410a968 (commit) via ca2c13b710254f7c84154ecd720805d673f58792 (commit) via cf2c025a6c23120e35cf82b8b9ef3c3dd3a0d066 (commit) via 3835f6012fce888b9ac4648f708b91ba449b0944 (commit) from b5099d559d6c2a406d3b875b054c288c7e306bc7 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/7d0d237967b9232cf8c2ad30a26d3efd22... commit 7d0d237967b9232cf8c2ad30a26d3efd2224e957 Author: Xin Zhao <[email protected]> Date: Fri Jul 24 10:21:10 2015 -0500 Rename "non_empty_slots" to "num_active_slots". Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index 568e222..f53469c 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -310,7 +310,7 @@ static inline int MPIDI_CH3I_Win_create_target(MPID_Win * win_ptr, int target_ra t->target_rank = target_rank; if (slot->target_list_head == NULL) - win_ptr->non_empty_slots++; + win_ptr->num_active_slots++; /* Enqueue target into target list. */ MPL_DL_APPEND(slot->target_list_head, t); @@ -428,7 +428,7 @@ static inline int MPIDI_CH3I_Win_target_dequeue_and_free(MPID_Win * win_ptr, MPI MPIU_ERR_POP(mpi_errno); if (slot->target_list_head == NULL) - win_ptr->non_empty_slots--; + win_ptr->num_active_slots--; fn_exit: return mpi_errno; @@ -455,7 +455,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_targets_win(MPID_Win * win_ptr) } } - MPIU_Assert(win_ptr->non_empty_slots == 0); + MPIU_Assert(win_ptr->num_active_slots == 0); fn_exit: return mpi_errno; diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 741c758..11cb6d3 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -329,7 +329,8 @@ typedef struct MPIDI_Win_basic_info { enum MPIDI_RMA_states access_state; \ enum MPIDI_RMA_states exposure_state; \ } states; \ - int non_empty_slots; \ + int num_active_slots; /* specify number of slots that have \ + active targets */ \ int active_req_cnt; /* keep track of number of active requests in \ current epoch, i.e., number of issued but \ incomplete RMA operations. */ \ diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 2879572..820241e 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -399,7 +399,7 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ (*made_progress) = 0; - if (win_ptr->non_empty_slots == 0 || target == NULL || target->pending_op_list_head == NULL) + if (win_ptr->num_active_slots == 0 || target == NULL || target->pending_op_list_head == NULL) goto fn_exit; /* Issue out operations in the list. */ @@ -540,7 +540,7 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) (*made_progress) = 0; - if (win_ptr->non_empty_slots == 0) + if (win_ptr->num_active_slots == 0) goto fn_exit; /* FIXME: we should optimize the issuing pattern here. */ @@ -597,7 +597,7 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) /* If we are in an free_ops_before_completion, the window must be holding * up resources. If it isn't, we are in the wrong window and * incorrectly entered this function. */ - MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop"); + MPIU_ERR_CHKANDJUMP(win_ptr->num_active_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop"); /* make nonblocking progress once */ mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_ptr, &made_progress); @@ -699,7 +699,7 @@ int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr) /* If we are in an aggressive cleanup, the window must be holding * up resources. If it isn't, we are in the wrong window and * incorrectly entered this function. */ - MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop"); + MPIU_ERR_CHKANDJUMP(win_ptr->num_active_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop"); /* find the first target that has something to issue */ for (i = 0; i < win_ptr->num_slots; i++) { @@ -758,7 +758,7 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe /* If we are in an aggressive cleanup, the window must be holding * up resources. If it isn't, we are in the wrong window and * incorrectly entered this function. */ - MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanotarget"); + MPIU_ERR_CHKANDJUMP(win_ptr->num_active_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanotarget"); if (win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) { /* switch to window-wide protocol */ @@ -927,7 +927,7 @@ int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress) goto fn_exit; } - if (win_ptr->non_empty_slots == 0) + if (win_ptr->num_active_slots == 0) goto fn_exit; mpi_errno = issue_ops_win(win_ptr, &temp_progress); @@ -976,7 +976,7 @@ int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress) continue; } - if (win_ptr->non_empty_slots == 0) + if (win_ptr->num_active_slots == 0) continue; mpi_errno = issue_ops_win(win_ptr, &temp_progress); diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 6869571..41ed281 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -653,7 +653,7 @@ int MPID_Win_fence(int assert, MPID_Win * win_ptr) finish_fence: /* Make sure that all targets are freed. */ - MPIU_Assert(win_ptr->non_empty_slots == 0); + MPIU_Assert(win_ptr->num_active_slots == 0); MPIU_Assert(win_ptr->active_req_cnt == 0); @@ -967,7 +967,7 @@ int MPID_Win_complete(MPID_Win * win_ptr) MPIU_Assert(win_ptr->start_req == NULL); /* Make sure that all targets are freed. */ - MPIU_Assert(win_ptr->non_empty_slots == 0); + MPIU_Assert(win_ptr->num_active_slots == 0); MPIU_Assert(win_ptr->active_req_cnt == 0); @@ -1671,7 +1671,7 @@ int MPID_Win_unlock_all(MPID_Win * win_ptr) win_ptr->lock_all_assert = 0; /* Make sure that all targets are freed. */ - MPIU_Assert(win_ptr->non_empty_slots == 0); + MPIU_Assert(win_ptr->num_active_slots == 0); MPIU_Assert(win_ptr->active_req_cnt == 0); diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index c823752..2cc7e64 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -288,7 +288,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->shm_allocated = FALSE; (*win_ptr)->states.access_state = MPIDI_RMA_NONE; (*win_ptr)->states.exposure_state = MPIDI_RMA_NONE; - (*win_ptr)->non_empty_slots = 0; + (*win_ptr)->num_active_slots = 0; (*win_ptr)->active_req_cnt = 0; (*win_ptr)->fence_sync_req = MPI_REQUEST_NULL; (*win_ptr)->start_req = NULL; http://git.mpich.org/mpich.git/commitdiff/dbb7afb47e3e6f515f358986acb0a5a0d4... commit dbb7afb47e3e6f515f358986acb0a5a0d4944e42 Author: Xin Zhao <[email protected]> Date: Fri Jul 24 09:51:13 2015 -0500 Delete MPIDI_RMA_Win_list_t struct. MPIDI_RMA_Win_list_t struct is not necessary needed, it was created when we make the linked list of RMA windows. We can just add prev and next pointers inside window struct and delete this struct. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 4beb3bb..820b5f8 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -105,13 +105,7 @@ typedef struct MPIDI_RMA_Slot { struct MPIDI_RMA_Target *target_list_head; } MPIDI_RMA_Slot_t; -typedef struct MPIDI_RMA_Win_list { - MPID_Win *win_ptr; - struct MPIDI_RMA_Win_list *next; - struct MPIDI_RMA_Win_list *prev; -} MPIDI_RMA_Win_list_t; - -extern MPIDI_RMA_Win_list_t *MPIDI_RMA_Win_list_head; +extern MPID_Win *MPIDI_RMA_Win_list_head; typedef struct MPIDI_RMA_Target_lock_entry { struct MPIDI_RMA_Target_lock_entry *next; diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index d4a7161..741c758 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -354,6 +354,8 @@ typedef struct MPIDI_Win_basic_info { their completion during RMA \ epoch, but we need to clean \ them up in Win_free. */ \ + struct MPID_Win *prev; \ + struct MPID_Win *next; \ #ifdef MPIDI_CH3_WIN_DECL #define MPID_DEV_WIN_DECL \ diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 8272b0e..2879572 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -949,7 +949,7 @@ int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress) #define FCNAME MPIU_QUOTE(FUNCNAME) int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress) { - MPIDI_RMA_Win_list_t *win_elem; + MPID_Win *win_ptr; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; @@ -957,18 +957,17 @@ int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress) if (MPIDI_CH3I_num_active_issued_win == 0 && MPIDI_CH3I_num_passive_win == 0) goto fn_exit; - for (win_elem = MPIDI_RMA_Win_list_head; win_elem; win_elem = win_elem->next) { + for (win_ptr = MPIDI_RMA_Win_list_head; win_ptr; win_ptr = win_ptr->next) { int temp_progress = 0; int is_able_to_issue = 0; - if (win_elem->win_ptr->states.access_state == MPIDI_RMA_NONE || - win_elem->win_ptr->states.access_state == MPIDI_RMA_FENCE_GRANTED || - win_elem->win_ptr->states.access_state == MPIDI_RMA_PSCW_GRANTED) + if (win_ptr->states.access_state == MPIDI_RMA_NONE || + win_ptr->states.access_state == MPIDI_RMA_FENCE_GRANTED || + win_ptr->states.access_state == MPIDI_RMA_PSCW_GRANTED) continue; /* check and try to switch window state */ - mpi_errno = - check_and_switch_window_state(win_elem->win_ptr, &is_able_to_issue, &temp_progress); + mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) @@ -977,10 +976,10 @@ int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress) continue; } - if (win_elem->win_ptr->non_empty_slots == 0) + if (win_ptr->non_empty_slots == 0) continue; - mpi_errno = issue_ops_win(win_elem->win_ptr, &temp_progress); + mpi_errno = issue_ops_win(win_ptr, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index e426c3c..c823752 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -41,7 +41,7 @@ cvars: MPIU_THREADSAFE_INIT_DECL(initRMAoptions); -MPIDI_RMA_Win_list_t *MPIDI_RMA_Win_list_head = NULL; +MPID_Win *MPIDI_RMA_Win_list_head = NULL; static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, MPID_Info * info, MPID_Comm * comm_ptr, MPID_Win ** win_ptr); @@ -245,7 +245,6 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, int i; MPID_Comm *win_comm_ptr; int win_target_pool_size; - MPIDI_RMA_Win_list_t *win_elem; MPIU_CHKPMEM_DECL(5); MPIDI_STATE_DECL(MPID_STATE_WIN_INIT); @@ -300,6 +299,8 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->outstanding_locks = 0; (*win_ptr)->current_target_lock_data_bytes = 0; (*win_ptr)->dangling_request_cnt = 0; + (*win_ptr)->next = NULL; + (*win_ptr)->prev = NULL; /* Initialize the info flags */ (*win_ptr)->info_args.no_locks = 0; @@ -353,17 +354,12 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, &((*win_ptr)->target_lock_entry_pool_start[i])); } - /* enqueue window into the global list */ - MPIU_CHKPMEM_MALLOC(win_elem, MPIDI_RMA_Win_list_t *, sizeof(MPIDI_RMA_Win_list_t), mpi_errno, - "Window list element"); - win_elem->win_ptr = *win_ptr; - if (MPIDI_RMA_Win_list_head == NULL) { mpi_errno = MPID_Progress_register_hook(MPIDI_CH3I_RMA_Make_progress_global); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } - MPL_DL_APPEND(MPIDI_RMA_Win_list_head, win_elem); + MPL_DL_APPEND(MPIDI_RMA_Win_list_head, (*win_ptr)); if (MPIDI_CH3U_Win_hooks.win_init != NULL) { mpi_errno = diff --git a/src/mpid/ch3/src/mpidi_rma.c b/src/mpid/ch3/src/mpidi_rma.c index 4a01f90..7124be5 100644 --- a/src/mpid/ch3/src/mpidi_rma.c +++ b/src/mpid/ch3/src/mpidi_rma.c @@ -149,7 +149,6 @@ int MPID_Win_free(MPID_Win ** win_ptr) int in_use; MPID_Comm *comm_ptr; mpir_errflag_t errflag = MPIR_ERR_NONE; - MPIDI_RMA_Win_list_t *win_elem; MPIDI_STATE_DECL(MPID_STATE_MPID_WIN_FREE); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPID_WIN_FREE); @@ -201,11 +200,7 @@ int MPID_Win_free(MPID_Win ** win_ptr) } /* dequeue window from the global list */ - for (win_elem = MPIDI_RMA_Win_list_head; win_elem && win_elem->win_ptr != *win_ptr; - win_elem = win_elem->next); - MPIU_ERR_CHKANDJUMP(win_elem == NULL, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - MPL_DL_DELETE(MPIDI_RMA_Win_list_head, win_elem); - MPIU_Free(win_elem); + MPL_DL_DELETE(MPIDI_RMA_Win_list_head, (*win_ptr)); if (MPIDI_RMA_Win_list_head == NULL) MPID_Progress_deregister_hook(MPIDI_CH3I_RMA_Make_progress_global); http://git.mpich.org/mpich.git/commitdiff/e8a3c9d9178f95dc1f803e84982e56cf73... commit e8a3c9d9178f95dc1f803e84982e56cf739d3639 Author: Xin Zhao <[email protected]> Date: Sat Jul 18 10:44:20 2015 -0500 Add helper macros to simplify the code. Here we add three macros to simplify RMA code: (1) MPIDI_CH3I_RMA_PKT_IS_READ_OP: judging if the operation is a "read" op, which means has response data from target to origin. (2) MPIDI_CH3I_RMA_PKT_IS_IMMED_OP: judging if the operation is a "immed" op, which means all data fits in the packet header. (3) MPIDI_CH3I_RMA_Rank_to_slots_idx: provide mapping between rank and slot index. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index b55949a..568e222 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -24,6 +24,8 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); #define MPIDI_CH3I_RMA_Get_issued_list_ptr(target_, op_, list_ptr_, mpi_errno_) \ do { \ MPI_Datatype target_datatype_; \ + int is_read_; \ + MPIDI_CH3I_RMA_PKT_IS_READ_OP((op_)->pkt, is_read_); \ MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE((op_)->pkt, target_datatype_, mpi_errno_); \ if ((target_datatype_ != MPI_DATATYPE_NULL && \ !MPIR_DATATYPE_IS_PREDEFINED(target_datatype_)) || \ @@ -33,10 +35,7 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); !MPIR_DATATYPE_IS_PREDEFINED((op_)->result_datatype))) { \ list_ptr_ = &((target_)->issued_dt_op_list_head); \ } \ - else if ((op_)->pkt.type == MPIDI_CH3_PKT_PUT || \ - (op_)->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || \ - (op_)->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || \ - (op_)->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { \ + else if (!is_read_) { \ list_ptr_ = &((target_)->issued_write_op_list_head); \ } \ else { \ @@ -141,6 +140,14 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); \ } while (0) + +/* Given a rank, return slot index */ +#define MPIDI_CH3I_RMA_RANK_TO_SLOT(win_ptr_, rank_) \ + (((win_ptr_)->num_slots < (win_ptr_)->comm_ptr->local_size) ? \ + &(win_ptr_)->slots[(rank_) % (win_ptr_)->num_slots] : \ + &(win_ptr_)->slots[(rank_)]) + + /* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and * initialize it. If we cannot get one, return NULL. */ #undef FUNCNAME @@ -292,11 +299,7 @@ static inline int MPIDI_CH3I_Win_create_target(MPID_Win * win_ptr, int target_ra MPIDI_RMA_Slot_t *slot = NULL; MPIDI_RMA_Target_t *t = NULL; - if (win_ptr->num_slots < win_ptr->comm_ptr->local_size) - slot = &(win_ptr->slots[target_rank % win_ptr->num_slots]); - else - slot = &(win_ptr->slots[target_rank]); - + slot = MPIDI_CH3I_RMA_RANK_TO_SLOT(win_ptr, target_rank); t = MPIDI_CH3I_Win_target_alloc(win_ptr); if (t == NULL) { mpi_errno = MPIDI_CH3I_RMA_Cleanup_target_aggressive(win_ptr, &t); @@ -335,10 +338,7 @@ static inline int MPIDI_CH3I_Win_find_target(MPID_Win * win_ptr, int target_rank MPIDI_RMA_Slot_t *slot = NULL; MPIDI_RMA_Target_t *t = NULL; - if (win_ptr->num_slots < win_ptr->comm_ptr->local_size) - slot = &(win_ptr->slots[target_rank % win_ptr->num_slots]); - else - slot = &(win_ptr->slots[target_rank]); + slot = MPIDI_CH3I_RMA_RANK_TO_SLOT(win_ptr, target_rank); t = slot->target_list_head; while (t != NULL) { @@ -419,10 +419,7 @@ static inline int MPIDI_CH3I_Win_target_dequeue_and_free(MPID_Win * win_ptr, MPI int target_rank = e->target_rank; MPIDI_RMA_Slot_t *slot; - if (win_ptr->num_slots < win_ptr->comm_ptr->local_size) - slot = &(win_ptr->slots[target_rank % win_ptr->num_slots]); - else - slot = &(win_ptr->slots[target_rank]); + slot = MPIDI_CH3I_RMA_RANK_TO_SLOT(win_ptr, target_rank); MPL_DL_DELETE(slot->target_list_head, e); diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index db59ad7..05780e4 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -609,6 +609,41 @@ MPIDI_CH3_PKT_DEFS } \ } + +/* This macro judges if the RMA operation is a read operation, + * which means, it will triffer the issuing of response data from + * the target to the origin */ +#define MPIDI_CH3I_RMA_PKT_IS_READ_OP(pkt_, is_read_op_) \ + do { \ + if ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_GET_ACCUM || \ + (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_FOP || \ + (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_GET) { \ + is_read_op_ = TRUE; \ + } \ + else { \ + is_read_op_ = FALSE; \ + } \ + } while (0) + + +/* This macro judges if the RMA operation is a immed operation */ +#define MPIDI_CH3I_RMA_PKT_IS_IMMED_OP(pkt_, is_immed_op_) \ + do { \ + if ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_PUT_IMMED || \ + (pkt_).type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { \ + is_immed_op_ = TRUE; \ + } \ + else { \ + is_immed_op_ = FALSE; \ + } \ + } while (0) + typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 01a74c0..88a1eed 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -326,6 +326,7 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, MPIDI_CH3_Pkt_flags_t flag; MPI_Win source_win_handle; MPI_Request request_handle; + int is_immed_op = FALSE; int lock_discarded = 0, data_discarded = 0; int mpi_errno = MPI_SUCCESS; @@ -342,13 +343,8 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, lock_discarded = 1; } - if (pkt->type == MPIDI_CH3_PKT_LOCK || - pkt->type == MPIDI_CH3_PKT_PUT_IMMED || - pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || - pkt->type == MPIDI_CH3_PKT_GET || - pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || - pkt->type == MPIDI_CH3_PKT_FOP_IMMED || pkt->type == MPIDI_CH3_PKT_CAS_IMMED) { - + MPIDI_CH3I_RMA_PKT_IS_IMMED_OP((*pkt), is_immed_op); + if (is_immed_op || pkt->type == MPIDI_CH3_PKT_LOCK || pkt->type == MPIDI_CH3_PKT_GET) { /* return bytes of data processed in this pkt handler */ (*buflen) = sizeof(MPIDI_CH3_Pkt_t); @@ -429,10 +425,12 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t new_pkt; MPIDI_CH3_Pkt_lock_t *lock_pkt = &new_pkt.lock; MPI_Win target_win_handle; + int is_read_op = FALSE; + MPIDI_CH3I_RMA_PKT_IS_READ_OP((*pkt), is_read_op); MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE((*pkt), target_win_handle, mpi_errno); - if (pkt->type == MPIDI_CH3_PKT_PUT || pkt->type == MPIDI_CH3_PKT_ACCUMULATE) { + if (!is_read_op) { MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); request_handle = MPI_REQUEST_NULL; } @@ -527,6 +525,8 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, MPIU_ERR_POP(mpi_errno); } else { + int is_read_op = FALSE; + if (lock_discarded) flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED; else if (data_discarded) @@ -534,8 +534,8 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, else flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED; - if (pkt->type == MPIDI_CH3_PKT_PUT || pkt->type == MPIDI_CH3_PKT_PUT_IMMED || - pkt->type == MPIDI_CH3_PKT_ACCUMULATE || pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { + MPIDI_CH3I_RMA_PKT_IS_READ_OP((*pkt), is_read_op); + if (!is_read_op) { MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); request_handle = MPI_REQUEST_NULL; } diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 6ab1c8f..8272b0e 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -406,6 +406,7 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ curr_op = target->next_op_to_issue; while (curr_op != NULL) { int op_completed = FALSE; + int is_read_op = FALSE; if (target->access_state == MPIDI_RMA_LOCK_ISSUED) { /* It is possible that the previous OP+LOCK changes @@ -474,10 +475,8 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ (*made_progress) = 1; - if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || - curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { + MPIDI_CH3I_RMA_PKT_IS_READ_OP(curr_op->pkt, is_read_op); + if (!is_read_op) { target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending * PUT/ACC operation. */ } diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index cd6c415..6869571 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -903,7 +903,6 @@ int MPID_Win_complete(MPID_Win * win_ptr) { int mpi_errno = MPI_SUCCESS; int i, dst, rank = win_ptr->comm_ptr->rank; - MPID_Comm *win_comm_ptr = win_ptr->comm_ptr; MPIDI_RMA_Target_t *curr_target; MPIDI_STATE_DECL(MPID_STATE_MPID_WIN_COMPLETE); @@ -935,13 +934,9 @@ int MPID_Win_complete(MPID_Win * win_ptr) continue; } - if (win_comm_ptr->local_size <= win_ptr->num_slots) - curr_target = win_ptr->slots[dst].target_list_head; - else { - curr_target = win_ptr->slots[dst % win_ptr->num_slots].target_list_head; - while (curr_target != NULL && curr_target->target_rank != dst) - curr_target = curr_target->next; - } + curr_target = MPIDI_CH3I_RMA_RANK_TO_SLOT(win_ptr, dst)->target_list_head; + while (curr_target != NULL && curr_target->target_rank != dst) + curr_target = curr_target->next; if (curr_target != NULL) { curr_target->win_complete_flag = 1; @@ -1618,13 +1613,9 @@ int MPID_Win_unlock_all(MPID_Win * win_ptr) } else { for (i = 0; i < win_ptr->comm_ptr->local_size; i++) { - if (win_ptr->comm_ptr->local_size <= win_ptr->num_slots) - curr_target = win_ptr->slots[i].target_list_head; - else { - curr_target = win_ptr->slots[i % win_ptr->num_slots].target_list_head; - while (curr_target != NULL && curr_target->target_rank != i) - curr_target = curr_target->next; - } + curr_target = MPIDI_CH3I_RMA_RANK_TO_SLOT(win_ptr, i)->target_list_head; + while (curr_target != NULL && curr_target->target_rank != i) + curr_target = curr_target->next; if (curr_target != NULL) { if (curr_target->sync.sync_flag < sync_flag) { http://git.mpich.org/mpich.git/commitdiff/b7066037f8b539ef9979984d2ebdbaca03... commit b7066037f8b539ef9979984d2ebdbaca0322e1bf Author: Xin Zhao <[email protected]> Date: Fri Jul 24 09:32:50 2015 -0500 Simplify RMA issuing functions. For RMA operations that have response data back, we always keep track of the receiving request in the operation object and let the sending request go into the progress engine (it will be completed by the progress engine when sending is completed). Here we simplify this part of code. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 3a7b6d9..d36d74a 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -692,16 +692,9 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, if (curr_req != NULL) { MPID_Request_release(curr_req); - curr_req = resp_req; - } - else { - curr_req = resp_req; } - /* For error checking */ - resp_req = NULL; - - rma_op->single_req = curr_req; + rma_op->single_req = resp_req; goto fn_exit; } @@ -814,39 +807,14 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - /* This operation can generate two requests; one for inbound and one for - * outbound data. */ if (curr_req != NULL) { - /* If we have both inbound and outbound requests (i.e. GACC - * operation), we need to ensure that the source buffer is - * available and that the response data has been received before - * informing the origin that this operation is complete. Because - * the update needs to be done atomically at the target, they will - * not send back data until it has been received. Therefore, - * completion of the response request implies that the send request - * has completed. - * - * Therefore: refs on the response request are set to two: one is - * held by the progress engine and the other by the RMA op - * completion code. Refs on the outbound request are set to one; - * it will be completed by the progress engine. - */ - MPID_Request_release(curr_req); - curr_req = resp_req; - } - else { - curr_req = resp_req; } - /* For error checking */ - resp_req = NULL; - if (rma_op->reqs_size == 1) - rma_op->single_req = curr_req; + rma_op->single_req = resp_req; else - rma_op->multi_reqs[j] = curr_req; - + rma_op->multi_reqs[j] = resp_req; rma_op->issued_stream_count++; @@ -1140,35 +1108,11 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, MPIU_ERR_POP(mpi_errno); } - /* This operation can generate two requests; one for inbound and one for - * outbound data. */ if (curr_req != NULL) { - /* If we have both inbound and outbound requests (i.e. GACC - * operation), we need to ensure that the source buffer is - * available and that the response data has been received before - * informing the origin that this operation is complete. Because - * the update needs to be done atomically at the target, they will - * not send back data until it has been received. Therefore, - * completion of the response request implies that the send request - * has completed. - * - * Therefore: refs on the response request are set to two: one is - * held by the progress engine and the other by the RMA op - * completion code. Refs on the outbound request are set to one; - * it will be completed by the progress engine. - */ - MPID_Request_release(curr_req); - curr_req = resp_req; - } - else { - curr_req = resp_req; } - /* For error checking */ - resp_req = NULL; - - rma_op->single_req = curr_req; + rma_op->single_req = resp_req; fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_ISSUE_FOP_OP); http://git.mpich.org/mpich.git/commitdiff/67bbc9a50f61aef5bf53fd07c84775f114... commit 67bbc9a50f61aef5bf53fd07c84775f11410a968 Author: Xin Zhao <[email protected]> Date: Fri Jul 24 09:33:19 2015 -0500 Simplify function dealing with operation piggybacked with LOCK. The operation that is entering this function must be the operation that piggybacked with LOCK, therefore we can simplify it by deleting unnecessary branches. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 4f86910..01a74c0 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -622,7 +622,6 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, MPIDI_RMA_Target_t *target = NULL; MPIDI_RMA_Op_t *op = NULL; MPIDI_CH3_Pkt_flags_t op_flags = MPIDI_CH3_PKT_FLAG_NONE; - int i; int mpi_errno = MPI_SUCCESS; mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, target_rank, &target); @@ -630,93 +629,81 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, MPIU_ERR_POP(mpi_errno); MPIU_Assert(target != NULL); - op = target->pending_op_list_head; - if (op != NULL) - MPIDI_CH3_PKT_RMA_GET_FLAGS(op->pkt, op_flags, mpi_errno); + /* Here the next_op_to_issue pointer should still point to the OP piggybacked + * with LOCK */ + op = target->next_op_to_issue; + MPIU_Assert(op != NULL); - if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || - op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED || - flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED) { + MPIDI_CH3_PKT_RMA_GET_FLAGS(op->pkt, op_flags, mpi_errno); + MPIU_Assert(op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); - if (op->ureq != NULL) { - mpi_errno = set_user_req_after_issuing_op(op); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - } + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED) { - if (op->reqs_size == 0) { - MPIU_Assert(op->single_req == NULL && op->multi_reqs == NULL); - MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head), op); - } - else { - MPI_Datatype target_datatype; - int is_derived = FALSE; + if (op->reqs_size > 0) { + MPID_Request **req = NULL; + if (op->reqs_size == 1) + req = &(op->single_req); + else + req = &(op->multi_reqs[0]); - MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head), op); + MPIU_Assert((*req) != NULL); + MPID_Request_release((*req)); + (*req) = NULL; + } - MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(op->pkt, target_datatype, mpi_errno); + if ((op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) + && op->issued_stream_count != ALL_STREAM_UNITS_ISSUED) { + /* Now we successfully issue out the first stream unit, + * keep next_op_to_issue still stick to the current op + * since we need to issue the following stream units. */ + goto fn_exit; + } - if ((target_datatype != MPI_DATATYPE_NULL && - !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) || - (op->origin_datatype != MPI_DATATYPE_NULL && - !MPIR_DATATYPE_IS_PREDEFINED(op->origin_datatype)) || - (op->result_datatype != MPI_DATATYPE_NULL && - !MPIR_DATATYPE_IS_PREDEFINED(op->result_datatype))) { - is_derived = TRUE; - } + /* We are done with the current operation, make next_op_to_issue points to the + * next operation. */ + target->next_op_to_issue = op->next; - if (is_derived) { - MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head), op); - } - else if (op->pkt.type == MPIDI_CH3_PKT_PUT || - op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || - op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || - op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { - MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head), op); - } - else { - MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head), op); - } + if (target->next_op_to_issue == NULL) { + if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || op_flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { + /* We are done with ending sync, unset target's sync_flag. */ + target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; } } - else if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED || - flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED) { - /* We need to re-transmit this operation, so we destroy - * the internal request and erase all flags in current - * operation. */ - if (op->reqs_size == 1) { - MPIU_Assert(op->single_req != NULL); - MPID_Request_release(op->single_req); - op->single_req = NULL; - win_ptr->active_req_cnt--; - op->reqs_size = 0; - } - else if (op->reqs_size > 1) { - MPIU_Assert(op->multi_reqs != NULL); - for (i = 0; i < op->reqs_size; i++) { - if (op->multi_reqs[i] != NULL) { - MPID_Request_release(op->multi_reqs[i]); - op->multi_reqs[i] = NULL; - win_ptr->active_req_cnt--; - } - } - /* free req array in this op */ - MPIU_Free(op->multi_reqs); - op->multi_reqs = NULL; - op->reqs_size = 0; - } - MPIDI_CH3_PKT_RMA_ERASE_FLAGS(op->pkt, mpi_errno); - - target->next_op_to_issue = op; - op->issued_stream_count = 0; + if (op->ureq != NULL) { + /* Complete user request and release the ch3 ref */ + mpi_errno = MPID_Request_complete(op->ureq); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + } - if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) - target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - else if (op_flags & MPIDI_RMA_SYNC_UNLOCK) - target->sync.sync_flag = MPIDI_RMA_SYNC_UNLOCK; + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head), op); + } + else if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED) { + /* We need to re-transmit this operation, so we destroy + * the internal request and erase all flags in current + * operation. */ + if (op->reqs_size == 1) { + MPIU_Assert(op->single_req != NULL); + MPID_Request_release(op->single_req); + op->single_req = NULL; + op->reqs_size = 0; } + else if (op->reqs_size > 1) { + MPIU_Assert(op->multi_reqs != NULL && op->multi_reqs[0] != NULL); + MPID_Request_release(op->multi_reqs[0]); + /* free req array in this op */ + MPIU_Free(op->multi_reqs); + op->multi_reqs = NULL; + op->reqs_size = 0; + } + MPIDI_CH3_PKT_RMA_ERASE_FLAGS(op->pkt, mpi_errno); + + op->issued_stream_count = 0; } fn_exit: http://git.mpich.org/mpich.git/commitdiff/ca2c13b710254f7c84154ecd720805d673... commit ca2c13b710254f7c84154ecd720805d673f58792 Author: Xin Zhao <[email protected]> Date: Fri Jul 24 09:30:49 2015 -0500 Add bracket to protect variable name in the macro. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index bc9fd90..b55949a 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -50,23 +50,23 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); do { \ local_completed_ = 0; \ remote_completed_ = 0; \ - if (win_->states.access_state != MPIDI_RMA_FENCE_ISSUED && \ - win_->states.access_state != MPIDI_RMA_PSCW_ISSUED && \ - win_->states.access_state != MPIDI_RMA_LOCK_ALL_ISSUED && \ - target_->access_state != MPIDI_RMA_LOCK_CALLED && \ - target_->access_state != MPIDI_RMA_LOCK_ISSUED && \ - target_->pending_op_list_head == NULL && \ - target_->issued_read_op_list_head == NULL && \ - target_->issued_write_op_list_head == NULL && \ - target_->issued_dt_op_list_head == NULL) { \ + if ((win_)->states.access_state != MPIDI_RMA_FENCE_ISSUED && \ + (win_)->states.access_state != MPIDI_RMA_PSCW_ISSUED && \ + (win_)->states.access_state != MPIDI_RMA_LOCK_ALL_ISSUED && \ + (target_)->access_state != MPIDI_RMA_LOCK_CALLED && \ + (target_)->access_state != MPIDI_RMA_LOCK_ISSUED && \ + (target_)->pending_op_list_head == NULL && \ + (target_)->issued_read_op_list_head == NULL && \ + (target_)->issued_write_op_list_head == NULL && \ + (target_)->issued_dt_op_list_head == NULL) { \ local_completed_ = 1; \ - if (target_->sync.sync_flag == MPIDI_RMA_SYNC_NONE && \ - target_->sync.outstanding_acks == 0) \ + if ((target_)->sync.sync_flag == MPIDI_RMA_SYNC_NONE && \ + (target_)->sync.outstanding_acks == 0) \ remote_completed_ = 1; \ } \ \ - if ((target_->sync.upgrade_flush_local && !remote_completed_) || \ - (!target_->sync.upgrade_flush_local && !local_completed_)) { \ + if (((target_)->sync.upgrade_flush_local && !remote_completed_) || \ + (!(target_)->sync.upgrade_flush_local && !local_completed_)) { \ local_completed_ = 0; \ } \ else { \ http://git.mpich.org/mpich.git/commitdiff/cf2c025a6c23120e35cf82b8b9ef3c3dd3... commit cf2c025a6c23120e35cf82b8b9ef3c3dd3a0d066 Author: Xin Zhao <[email protected]> Date: Fri Jul 24 09:26:30 2015 -0500 Make request completion callback to delete operation and drop GC progress. In this patch, we make the request point to the corresponding RMA operation and delete that operation from the list in the request completion callback, so that operation can be deleted as soon as possible. By doing so, the garbage collection progress of RMA operations is not needed anymore. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index c9e01a3..3a7b6d9 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -453,7 +453,6 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, rma_op->reqs_size = 1; rma_op->single_req = curr_req; - win_ptr->active_req_cnt++; } fn_exit: @@ -512,7 +511,6 @@ static int issue_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, rma_op->reqs_size = 1; rma_op->single_req = curr_req; - win_ptr->active_req_cnt++; } goto fn_exit; } @@ -601,7 +599,6 @@ static int issue_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, rma_op->single_req = curr_req; else rma_op->multi_reqs[j] = curr_req; - win_ptr->active_req_cnt++; } rma_op->issued_stream_count++; @@ -705,7 +702,6 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, resp_req = NULL; rma_op->single_req = curr_req; - win_ptr->active_req_cnt++; goto fn_exit; } @@ -851,7 +847,6 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, else rma_op->multi_reqs[j] = curr_req; - win_ptr->active_req_cnt++; rma_op->issued_stream_count++; @@ -1014,7 +1009,6 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, } rma_op->single_req = curr_req; - win_ptr->active_req_cnt++; fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_ISSUE_GET_OP); @@ -1078,7 +1072,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, } rma_op->single_req = curr_req; - win_ptr->active_req_cnt++; fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_ISSUE_CAS_OP); @@ -1176,7 +1169,6 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, resp_req = NULL; rma_op->single_req = curr_req; - win_ptr->active_req_cnt++; fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_ISSUE_FOP_OP); @@ -1243,83 +1235,4 @@ static inline int issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, /* --END ERROR HANDLING-- */ } -#undef FUNCNAME -#define FUNCNAME set_user_req_after_issuing_op -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op) -{ - int i, incomplete_req_cnt = 0; - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_SET_USER_REQ_AFTER_ISSUING_OP); - - MPIDI_RMA_FUNC_ENTER(MPID_STATE_SET_USER_REQ_AFTER_ISSUING_OP); - - if (op->ureq == NULL) - goto fn_exit; - - if (op->reqs_size == 0) { - MPIU_Assert(op->single_req == NULL && op->multi_reqs == NULL); - /* Sending is completed immediately, complete user request - * and release ch3 ref. */ - - /* Complete user request and release the ch3 ref */ - mpi_errno = MPID_Request_complete(op->ureq); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_POP(mpi_errno); - } - } - else { - MPID_Request **req_ptr = NULL; - - /* Sending is not completed immediately. */ - - if (op->reqs_size == 1) - req_ptr = &(op->single_req); - else - req_ptr = op->multi_reqs; - - for (i = 0; i < op->reqs_size; i++) { - if (req_ptr[i] == NULL || MPID_Request_is_complete(req_ptr[i])) - continue; - - /* Setup user request info in order to be completed following send request. */ - incomplete_req_cnt++; - MPID_cc_set(&(op->ureq->cc), incomplete_req_cnt); /* increment CC counter */ - - req_ptr[i]->dev.request_handle = op->ureq->handle; - - /* Setup user request completion handler. - * - * The handler is triggered when send request is completed at - * following places: - * - progress engine: complete PUT/ACC req. - * - GET/GET_ACC packet handler: complete GET/GET_ACC reqs. - * - * TODO: implement stack for overriding functions*/ - req_ptr[i]->request_completed_cb = MPIDI_CH3_ReqHandler_ReqOpsComplete; - } /* end of for loop */ - - if (incomplete_req_cnt) { - /* Increase ref for completion handler */ - MPIU_Object_add_ref(op->ureq); - } - else { - /* all requests are completed */ - /* Complete user request and release ch3 ref */ - mpi_errno = MPID_Request_complete(op->ureq); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_POP(mpi_errno); - } - op->ureq = NULL; - } - } - - fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_SET_USER_REQ_AFTER_ISSUING_OP); - return mpi_errno; - fn_fail: - goto fn_exit; -} - #endif /* MPID_RMA_ISSUE_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index 5d84d6f..bc9fd90 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -21,6 +21,31 @@ extern MPIDI_RMA_Target_t *global_rma_target_pool_head, *global_rma_target_pool_ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); +#define MPIDI_CH3I_RMA_Get_issued_list_ptr(target_, op_, list_ptr_, mpi_errno_) \ + do { \ + MPI_Datatype target_datatype_; \ + MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE((op_)->pkt, target_datatype_, mpi_errno_); \ + if ((target_datatype_ != MPI_DATATYPE_NULL && \ + !MPIR_DATATYPE_IS_PREDEFINED(target_datatype_)) || \ + ((op_)->origin_datatype != MPI_DATATYPE_NULL && \ + !MPIR_DATATYPE_IS_PREDEFINED((op_)->origin_datatype)) || \ + ((op_)->result_datatype != MPI_DATATYPE_NULL && \ + !MPIR_DATATYPE_IS_PREDEFINED((op_)->result_datatype))) { \ + list_ptr_ = &((target_)->issued_dt_op_list_head); \ + } \ + else if ((op_)->pkt.type == MPIDI_CH3_PKT_PUT || \ + (op_)->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || \ + (op_)->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || \ + (op_)->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { \ + list_ptr_ = &((target_)->issued_write_op_list_head); \ + } \ + else { \ + list_ptr_ = &((target_)->issued_read_op_list_head); \ + } \ + } while (0) + +/* This macro returns two flags: local_completed and remote_completed, + * to indicate if the completion is reached on this target. */ #define MPIDI_CH3I_RMA_ops_completion(win_, target_, local_completed_, remote_completed_) \ do { \ local_completed_ = 0; \ @@ -39,6 +64,81 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); target_->sync.outstanding_acks == 0) \ remote_completed_ = 1; \ } \ + \ + if ((target_->sync.upgrade_flush_local && !remote_completed_) || \ + (!target_->sync.upgrade_flush_local && !local_completed_)) { \ + local_completed_ = 0; \ + } \ + else { \ + local_completed_ = 1; \ + } \ + \ + } while (0) + + +/* This macro returns a flag: win_remote_completed, to indicate if + * the remote completion is reached on the entire window. */ +#define MPIDI_CH3I_RMA_ops_win_remote_completion(win_ptr_, win_remote_completed_) \ + do { \ + MPIDI_RMA_Target_t *win_target_ = NULL; \ + int i_, num_targets_ = 0; \ + int remote_completed_targets_ = 0; \ + \ + win_remote_completed_ = 0; \ + \ + for (i_ = 0; i_ < (win_ptr_)->num_slots; i_++) { \ + for (win_target_ = (win_ptr_)->slots[i_].target_list_head; win_target_;) { \ + int local_ ATTRIBUTE((unused)) = 0, remote_ = 0; \ + \ + num_targets_++; \ + \ + MPIDI_CH3I_RMA_ops_completion((win_ptr_), win_target_, local_, remote_); \ + \ + remote_completed_targets_ += remote_; \ + \ + win_target_ = win_target_->next; \ + } \ + } \ + \ + if (num_targets_ == remote_completed_targets_) \ + win_remote_completed_ = 1; \ + \ + } while (0) + +/* This macro returns a flag: win_local_completed, to indicate if + * the local completion is reached on the entire window. */ +#define MPIDI_CH3I_RMA_ops_win_local_completion(win_ptr_, win_local_completed_) \ + do { \ + MPIDI_RMA_Target_t *win_target_ = NULL; \ + int i_, total_remote_cnt_ = 0, total_local_cnt_ = 0; \ + int remote_completed_targets_ = 0, local_completed_targets_ = 0; \ + \ + win_local_completed_ = 0; \ + \ + for (i_ = 0; i_ < (win_ptr_)->num_slots; i_++) { \ + for (win_target_ = (win_ptr_)->slots[i_].target_list_head; win_target_;) { \ + int local_ = 0, remote_ = 0; \ + \ + if (win_target_->sync.upgrade_flush_local) \ + total_remote_cnt_++; \ + else \ + total_local_cnt_++; \ + \ + MPIDI_CH3I_RMA_ops_completion((win_ptr_), win_target_, local_, remote_); \ + \ + if (win_target_->sync.upgrade_flush_local) \ + remote_completed_targets_ += remote_; \ + else \ + local_completed_targets_ += local_; \ + \ + win_target_ = win_target_->next; \ + } \ + } \ + \ + if (remote_completed_targets_ == total_remote_cnt_ && \ + local_completed_targets_ == total_local_cnt_) \ + win_local_completed_ = 1; \ + \ } while (0) /* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and @@ -75,6 +175,8 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr) e->origin_datatype = MPI_DATATYPE_NULL; e->result_datatype = MPI_DATATYPE_NULL; + e->ref_cnt = 0; + return e; } @@ -88,6 +190,10 @@ static inline int MPIDI_CH3I_Win_op_free(MPID_Win * win_ptr, MPIDI_RMA_Op_t * e) { int mpi_errno = MPI_SUCCESS; + if (e->multi_reqs != NULL) { + MPIU_Free(e->multi_reqs); + } + /* We enqueue elements to the right pool, so when they get freed * at window free time, they won't conflict with the global pool * or other windows */ @@ -333,193 +439,6 @@ static inline int MPIDI_CH3I_Win_target_dequeue_and_free(MPID_Win * win_ptr, MPI goto fn_exit; } - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_target -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target) -{ - MPIDI_RMA_Op_t *curr_op = NULL; - MPIDI_RMA_Op_t **op_list_head = NULL; - int read_flag = 0, write_flag = 0; - int mpi_errno = MPI_SUCCESS; - int i; - - if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) - goto fn_exit; - - if (target == NULL) - goto fn_exit; - - if (target->access_state == MPIDI_RMA_LOCK_CALLED || - target->access_state == MPIDI_RMA_LOCK_ISSUED) - goto fn_exit; - - if (target->pending_op_list_head == NULL && - target->issued_read_op_list_head == NULL && target->issued_write_op_list_head == NULL && - target->issued_dt_op_list_head == NULL) - goto fn_exit; - - /* go over issued_read_op_list, issued_write_op_list, - * issued_dt_op_list, start from issued_read_op_list. */ - op_list_head = &(target->issued_read_op_list_head); - read_flag = 1; - - curr_op = *op_list_head; - while (1) { - if (curr_op != NULL) { - int completed = 0; - - MPIU_Assert(curr_op->reqs_size > 0); - if (curr_op->reqs_size == 1) { - /* single_req is used */ - - if (MPID_Request_is_complete(curr_op->single_req)) { - /* If there's an error, return it */ - mpi_errno = curr_op->single_req->status.MPI_ERROR; - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - /* No errors, free the request */ - MPID_Request_release(curr_op->single_req); - - curr_op->single_req = NULL; - - win_ptr->active_req_cnt--; - - completed = 1; - } - else - break; - } - else { - /* multi_reqs is used */ - for (i = 0; i < curr_op->reqs_size; i++) { - if (curr_op->multi_reqs[i] == NULL) - continue; - - if (MPID_Request_is_complete(curr_op->multi_reqs[i])) { - /* If there's an error, return it */ - mpi_errno = curr_op->multi_reqs[i]->status.MPI_ERROR; - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); - - /* No errors, free the request */ - MPID_Request_release(curr_op->multi_reqs[i]); - - curr_op->multi_reqs[i] = NULL; - - win_ptr->active_req_cnt--; - } - else - break; - } - - if (i == curr_op->reqs_size) - completed = 1; - } - - if (completed) { - /* Release user request */ - if (curr_op->ureq) { - /* User request must be completed by progress engine */ - MPIU_Assert(MPID_Request_is_complete(curr_op->ureq)); - - /* Release the ch3 ref */ - MPID_Request_release(curr_op->ureq); - } - - if (curr_op->reqs_size == 1) { - curr_op->single_req = NULL; - } - else { - /* free request array in op struct */ - MPIU_Free(curr_op->multi_reqs); - curr_op->multi_reqs = NULL; - } - curr_op->reqs_size = 0; - - /* dequeue the operation and free it */ - MPL_DL_DELETE(*op_list_head, curr_op); - MPIDI_CH3I_Win_op_free(win_ptr, curr_op); - } - else - break; - } - else { - /* current op ptr reaches NULL, move on to the next list */ - if (read_flag == 1) { - read_flag = 0; - op_list_head = &(target->issued_write_op_list_head); - write_flag = 1; - } - else if (write_flag == 1) { - write_flag = 0; - op_list_head = &(target->issued_dt_op_list_head); - } - else { - /* we reach the tail of the last operation list (dt_op_list), - * break out. */ - break; - } - } - - /* next op */ - curr_op = *op_list_head; - } - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_win -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Cleanup_ops_win(MPID_Win * win_ptr, - int *local_completed, int *remote_completed) -{ - MPIDI_RMA_Target_t *target = NULL; - int num_targets = 0, local_completed_targets = 0, remote_completed_targets = 0; - int i, mpi_errno = MPI_SUCCESS; - - (*local_completed) = 0; - (*remote_completed) = 0; - - for (i = 0; i < win_ptr->num_slots; i++) { - for (target = win_ptr->slots[i].target_list_head; target;) { - int local = 0, remote = 0; - - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - - MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local, remote); - - num_targets++; - local_completed_targets += local; - remote_completed_targets += remote; - - target = target->next; - } - } - - if (num_targets == local_completed_targets) - (*local_completed) = 1; - if (num_targets == remote_completed_targets) - (*remote_completed) = 1; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - - #undef FUNCNAME #define FUNCNAME MPIDI_CH3I_RMA_Cleanup_targets_win #undef FCNAME @@ -554,7 +473,6 @@ static inline int MPIDI_CH3I_RMA_Cleanup_targets_win(MPID_Win * win_ptr) static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t ** e) { MPIDI_RMA_Op_t *new_ptr = NULL; - int local_completed = 0, remote_completed = 0; int mpi_errno = MPI_SUCCESS; while (1) { @@ -564,10 +482,6 @@ static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t ** e) if (new_ptr != NULL) break; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr); MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index ba664b1..4beb3bb 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -60,6 +60,8 @@ typedef struct MPIDI_RMA_Op { * when < 0, it means all stream units of this operation haven been issued. */ MPID_Request *ureq; + + int ref_cnt; } MPIDI_RMA_Op_t; typedef struct MPIDI_RMA_Target { diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 8f6c7e3..0f7f2f4 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1872,8 +1872,8 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *, MPID_Request *, int * ); int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *, MPID_Request *, int * ); -/* Request-based operation handler */ -int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPID_Request *); +/* RMA operation request handler */ +int MPIDI_CH3_Req_handler_rma_op_complete(MPID_Request *); /* RMA Synchronization request handler */ int MPIDI_CH3_Req_handler_rma_sync_complete(MPID_Request *); diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 2f7d3ea..d4a7161 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -451,6 +451,8 @@ typedef struct MPIDI_Request { * and freed when release request. */ MPIDI_msg_sz_t ext_hdr_sz; + struct MPIDI_RMA_Op *rma_op_ptr; + MPIDI_REQUEST_SEQNUM /* Occasionally, when a message cannot be sent, we need to cache the diff --git a/src/mpid/ch3/src/ch3u_handle_op_req.c b/src/mpid/ch3/src/ch3u_handle_op_req.c index 822cea4..4315cf8 100644 --- a/src/mpid/ch3/src/ch3u_handle_op_req.c +++ b/src/mpid/ch3/src/ch3u_handle_op_req.c @@ -7,33 +7,81 @@ #include "mpidimpl.h" #include "mpidrma.h" -/* Note the the following function is called when request-based RMA operation - is completed at origin side. Here we complete the user request associated - with this request-based operation. */ + #undef FUNCNAME -#define FUNCNAME MPIDI_CH3_ReqHandler_ReqOpsComplete +#define FUNCNAME MPIDI_CH3_Req_handler_rma_op_complete #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -int MPIDI_CH3_ReqHandler_ReqOpsComplete(MPID_Request * sreq) +int MPIDI_CH3_Req_handler_rma_op_complete(MPID_Request * sreq) { - int mpi_errno = MPI_SUCCESS; + int i, mpi_errno = MPI_SUCCESS; MPID_Request *ureq = NULL; + MPIDI_RMA_Op_t *op = NULL; + MPID_Win *win_ptr = NULL; + + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_OP_COMPLETE); + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_OP_COMPLETE); + + /* get window, decrement active request cnt on window */ + MPID_Win_get_ptr(sreq->dev.source_win_handle, win_ptr); + MPIU_Assert(win_ptr != NULL); + win_ptr->active_req_cnt--; + + if (sreq->dev.request_handle != MPI_REQUEST_NULL) { + /* get user request */ + MPID_Request_get_ptr(sreq->dev.request_handle, ureq); + mpi_errno = MPID_Request_complete(ureq); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + } + + op = sreq->dev.rma_op_ptr; + /* Note: if rma_op_ptr is NULL, it means the operation + * is freed before the completion. */ + if (op != NULL) { + /* Mark this request as NULL in the operation struct, + * so that when operation is freed before completion, + * the operation do not need to try to notify this + * request which is already completed. */ + if (op->reqs_size == 1) { + MPIU_Assert(op->single_req->handle == sreq->handle); + op->single_req = NULL; + } + else { + for (i = 0; i < op->reqs_size; i++) { + if (op->multi_reqs[i] == NULL) + continue; + else if (op->multi_reqs[i]->handle == sreq->handle) { + op->multi_reqs[i] = NULL; + break; + } + } + } + + op->ref_cnt--; + MPIU_Assert(op->ref_cnt >= 0); + if (op->ref_cnt == 0) { + MPIDI_RMA_Target_t *target = NULL; + MPIDI_RMA_Op_t **list_ptr = NULL; - MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_REQOPSCOMPLETE); - MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_REQOPSCOMPLETE); + mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, op->target_rank, &target); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + MPIU_Assert(target != NULL); - MPID_Request_get_ptr(sreq->dev.request_handle, ureq); - MPIU_Assert(ureq != NULL); + MPIDI_CH3I_RMA_Get_issued_list_ptr(target, op, list_ptr, mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } - /* Complete user request and release ref of completion handler. - * Note that ch3 ref is released by later clean_up call. */ - mpi_errno = MPID_Request_complete(ureq); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_POP(mpi_errno); + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, list_ptr, op); + } } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_REQOPSCOMPLETE); + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQ_HANDLER_RMA_OP_COMPLETE); return mpi_errno; fn_fail: diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c index db8c7f1..94dbb91 100644 --- a/src/mpid/ch3/src/ch3u_request.c +++ b/src/mpid/ch3/src/ch3u_request.c @@ -100,6 +100,7 @@ MPID_Request * MPID_Request_create(void) req->dev.tmpbuf = NULL; req->dev.ext_hdr_ptr = NULL; req->dev.ext_hdr_sz = 0; + req->dev.rma_op_ptr = NULL; req->dev.request_handle = MPI_REQUEST_NULL; #ifdef MPIDI_CH3_REQUEST_INIT MPIDI_CH3_REQUEST_INIT(req); diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 2d2ea4b..65a2eb0 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -194,11 +194,6 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { - int local_completed = 0, remote_completed = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -368,11 +363,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { - int local_completed = 0, remote_completed = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -584,11 +574,6 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { - int local_completed = 0, remote_completed = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -841,11 +826,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { - int local_completed = 0, remote_completed = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1093,11 +1073,6 @@ int MPID_Compare_and_swap(const void *origin_addr, const void *compare_addr, if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { - int local_completed = 0, remote_completed = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1240,11 +1215,6 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { - int local_completed = 0, remote_completed = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index da604d4..6ab1c8f 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -309,6 +309,84 @@ static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Ta #undef FUNCNAME +#define FUNCNAME check_and_set_req_completion +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +static inline int check_and_set_req_completion(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, + MPIDI_RMA_Op_t * rma_op, int *op_completed) +{ + int i, mpi_errno = MPI_SUCCESS; + MPID_Request **req = NULL; + MPIDI_STATE_DECL(MPID_STATE_CHECK_AND_SET_REQ_COMPLETION); + + MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_AND_SET_REQ_COMPLETION); + + (*op_completed) = FALSE; + + if (rma_op->reqs_size > 0) { + for (i = 0; i < rma_op->reqs_size; i++) { + if (rma_op->reqs_size == 1) + req = &(rma_op->single_req); + else + req = &(rma_op->multi_reqs[i]); + + if ((*req) == NULL) + continue; + + if (MPID_Request_is_complete((*req))) { + MPID_Request_release((*req)); + (*req) = NULL; + } + else { + MPID_Request_release((*req)); + (*req)->request_completed_cb = MPIDI_CH3_Req_handler_rma_op_complete; + (*req)->dev.source_win_handle = win_ptr->handle; + (*req)->dev.rma_op_ptr = rma_op; + + rma_op->ref_cnt++; + + if (rma_op->ureq != NULL) { + MPID_cc_set(&(rma_op->ureq->cc), rma_op->ref_cnt); + (*req)->dev.request_handle = rma_op->ureq->handle; + } + } + } + } + + if (rma_op->ref_cnt == 0) { + if (rma_op->ureq != NULL) { + mpi_errno = MPID_Request_complete(rma_op->ureq); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + } + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head), rma_op); + + (*op_completed) = TRUE; + } + else { + MPIDI_RMA_Op_t **list_ptr = NULL; + + MPIDI_CH3I_RMA_Get_issued_list_ptr(target, rma_op, list_ptr, mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + + MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head), rma_op); + MPIDI_CH3I_RMA_Ops_append(list_ptr, rma_op); + + win_ptr->active_req_cnt += rma_op->ref_cnt; + } + + fn_exit: + MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_AND_SET_REQ_COMPLETION); + return mpi_errno; + fn_fail: + goto fn_exit; +} + + +#undef FUNCNAME #define FUNCNAME issue_ops_target #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) @@ -327,6 +405,7 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ /* Issue out operations in the list. */ curr_op = target->next_op_to_issue; while (curr_op != NULL) { + int op_completed = FALSE; if (target->access_state == MPIDI_RMA_LOCK_ISSUED) { /* It is possible that the previous OP+LOCK changes @@ -403,24 +482,6 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ * PUT/ACC operation. */ } - if ((curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || - curr_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) && - curr_op->issued_stream_count != ALL_STREAM_UNITS_ISSUED) { - /* For ACC-like operations, if not all stream units - * are issued out, we stick to the current operation, - * otherwise we move on to the next operation. */ - target->next_op_to_issue = curr_op; - } - else - target->next_op_to_issue = curr_op->next; - - if (target->next_op_to_issue == NULL) { - if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { - /* We are done with ending sync, unset target's sync_flag. */ - target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; - } - } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { /* If this operation is piggybacked with LOCK, @@ -430,63 +491,28 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ break; } - if (curr_op->ureq != NULL) { - mpi_errno = set_user_req_after_issuing_op(curr_op); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - } - - if (curr_op->reqs_size == 0) { - MPIU_Assert(curr_op->single_req == NULL && curr_op->multi_reqs == NULL); - /* Sending is completed immediately. */ - MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head), curr_op); - } - else { - MPI_Datatype target_datatype; - int is_derived = FALSE; - - /* Sending is not completed immediately. */ + target->next_op_to_issue = curr_op->next; - MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head), curr_op); - - MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno); - - if ((target_datatype != MPI_DATATYPE_NULL && - !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) || - (curr_op->origin_datatype != MPI_DATATYPE_NULL && - !MPIR_DATATYPE_IS_PREDEFINED(curr_op->origin_datatype)) || - (curr_op->result_datatype != MPI_DATATYPE_NULL && - !MPIR_DATATYPE_IS_PREDEFINED(curr_op->result_datatype))) { - is_derived = TRUE; - } - - if (is_derived) { - MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head), curr_op); - } - else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || - curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { - MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head), curr_op); - } - else { - MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head), curr_op); + if (target->next_op_to_issue == NULL) { + if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { + /* We are done with ending sync, unset target's sync_flag. */ + target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; } + } + mpi_errno = check_and_set_req_completion(win_ptr, target, curr_op, &op_completed); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + if (op_completed == FALSE) { /* Poke the progress engine when next_op_to_issue is not the current OP, in * order to make sure the issuing function is re-entrant safe. */ if (target->next_op_to_issue != curr_op && win_ptr->active_req_cnt > MPIR_CVAR_CH3_RMA_POKE_PROGRESS_REQ_THRESHOLD) { - int local_completed, remote_completed; mpi_errno = poke_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, - &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); } } @@ -622,27 +648,21 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) /* free all ops in the list since we do not need to maintain them anymore */ while (1) { if (curr_op != NULL) { + /* Here we mark the operation pointer in the request + * as NULL, so that when request is completed, it will + * not try to free this operation. */ if (curr_op->reqs_size == 1) { MPIU_Assert(curr_op->single_req != NULL); - MPID_Request_release(curr_op->single_req); - curr_op->single_req = NULL; - win_ptr->active_req_cnt--; - curr_op->reqs_size = 0; + curr_op->single_req->dev.rma_op_ptr = NULL; } - else if (curr_op->reqs_size > 1) { + else { + MPIU_Assert(curr_op->reqs_size > 1); MPIU_Assert(curr_op->multi_reqs != NULL); for (i = 0; i < curr_op->reqs_size; i++) { if (curr_op->multi_reqs[i] != NULL) { - MPID_Request_release(curr_op->multi_reqs[i]); - curr_op->multi_reqs[i] = NULL; - win_ptr->active_req_cnt--; + curr_op->multi_reqs[i]->dev.rma_op_ptr = NULL; } } - - /* free req array in this op */ - MPIU_Free(curr_op->multi_reqs); - curr_op->multi_reqs = NULL; - curr_op->reqs_size = 0; } MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, op_list_head, curr_op); } @@ -707,10 +727,6 @@ int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr) /* Wait for local completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed); if (!local_completed) { @@ -788,10 +804,6 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed); if (!remote_completed) { diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index e124931..cd6c415 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -332,10 +332,8 @@ void MPIDI_CH3_RMA_Init_sync_pvars(void) static inline int flush_local_all(MPID_Win * win_ptr) { int i, made_progress = 0; + int local_completed = 0; MPIDI_RMA_Target_t *curr_target = NULL; - int local_completed = 0, remote_completed = 0; - int total_remote_complete_cnt = 0, total_local_complete_cnt = 0; - int curr_remote_complete_cnt = 0, curr_local_complete_cnt = 0; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_FLUSH_LOCAL_ALL); @@ -349,13 +347,11 @@ static inline int flush_local_all(MPID_Win * win_ptr) if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; } - total_remote_complete_cnt++; } else { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL; } - total_local_complete_cnt++; } curr_target = curr_target->next; @@ -370,39 +366,14 @@ static inline int flush_local_all(MPID_Win * win_ptr) /* wait for remote completion for those targets that disable flush_local, * and wait for local completion for other targets */ do { - curr_local_complete_cnt = 0, curr_remote_complete_cnt = 0; - for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list_head; - while (curr_target != NULL) { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - - MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, - remote_completed); - - if (curr_target->sync.upgrade_flush_local) { - if (remote_completed) { - curr_remote_complete_cnt++; - } - } - else { - if (local_completed) { - curr_local_complete_cnt++; - } - } - curr_target = curr_target->next; - } - } + MPIDI_CH3I_RMA_ops_win_local_completion(win_ptr, local_completed); - if (curr_remote_complete_cnt < total_remote_complete_cnt || - curr_local_complete_cnt < total_local_complete_cnt) { + if (!local_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - } while (curr_remote_complete_cnt < total_remote_complete_cnt || - curr_local_complete_cnt < total_local_complete_cnt); + } while (!local_completed); fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_FLUSH_LOCAL_ALL); @@ -420,7 +391,7 @@ static inline int flush_local_all(MPID_Win * win_ptr) static inline int flush_all(MPID_Win * win_ptr) { int i, made_progress = 0; - int local_completed = 0, remote_completed = 0; + int remote_completed = 0; MPIDI_RMA_Target_t *curr_target = NULL; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_FLUSH_ALL); @@ -446,9 +417,7 @@ static inline int flush_all(MPID_Win * win_ptr) /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); + MPIDI_CH3I_RMA_ops_win_remote_completion(win_ptr, remote_completed); if (!remote_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) @@ -1267,10 +1236,6 @@ int MPID_Win_unlock(int dest, MPID_Win * win_ptr) /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local_completed, remote_completed); if (!remote_completed) { @@ -1377,10 +1342,6 @@ int MPID_Win_flush(int dest, MPID_Win * win_ptr) /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local_completed, remote_completed); if (!remote_completed) { @@ -1426,7 +1387,7 @@ int MPID_Win_flush(int dest, MPID_Win * win_ptr) int MPID_Win_flush_local(int dest, MPID_Win * win_ptr) { int made_progress = 0; - int local_completed = 0, remote_completed = 0; + int local_completed = 0, remote_completed ATTRIBUTE((unused)) = 0; int rank = win_ptr->comm_ptr->rank; MPIDI_RMA_Target_t *target = NULL; int mpi_errno = MPI_SUCCESS; @@ -1483,20 +1444,14 @@ int MPID_Win_flush_local(int dest, MPID_Win * win_ptr) /* Wait for local completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local_completed, remote_completed); - if ((target->sync.upgrade_flush_local && !remote_completed) || - (!target->sync.upgrade_flush_local && !local_completed)) { + if (!local_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - } while ((target->sync.upgrade_flush_local && !remote_completed) || - (!target->sync.upgrade_flush_local && !local_completed)); + } while (!local_completed); finish_flush_local: if (target != NULL) { @@ -1603,7 +1558,7 @@ int MPID_Win_lock_all(int assert, MPID_Win * win_ptr) int MPID_Win_unlock_all(MPID_Win * win_ptr) { int i, made_progress = 0; - int local_completed = 0, remote_completed = 0; + int remote_completed = 0; int rank = win_ptr->comm_ptr->rank; MPIDI_RMA_Target_t *curr_target = NULL; enum MPIDI_RMA_sync_types sync_flag; @@ -1703,9 +1658,7 @@ int MPID_Win_unlock_all(MPID_Win * win_ptr) /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, &remote_completed); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); + MPIDI_CH3I_RMA_ops_win_remote_completion(win_ptr, remote_completed); if (!remote_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) http://git.mpich.org/mpich.git/commitdiff/3835f6012fce888b9ac4648f708b91ba44... commit 3835f6012fce888b9ac4648f708b91ba449b0944 Author: Xin Zhao <[email protected]> Date: Fri Jul 24 09:27:05 2015 -0500 Add initialization of request_handle in the request object. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c index 53230f1..db8c7f1 100644 --- a/src/mpid/ch3/src/ch3u_request.c +++ b/src/mpid/ch3/src/ch3u_request.c @@ -100,6 +100,7 @@ MPID_Request * MPID_Request_create(void) req->dev.tmpbuf = NULL; req->dev.ext_hdr_ptr = NULL; req->dev.ext_hdr_sz = 0; + req->dev.request_handle = MPI_REQUEST_NULL; #ifdef MPIDI_CH3_REQUEST_INIT MPIDI_CH3_REQUEST_INIT(req); #endif ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/include/mpid_rma_issue.h | 151 +-------------- src/mpid/ch3/include/mpid_rma_oplist.h | 349 ++++++++++++-------------------- src/mpid/ch3/include/mpid_rma_types.h | 10 +- src/mpid/ch3/include/mpidimpl.h | 4 +- src/mpid/ch3/include/mpidpkt.h | 35 ++++ src/mpid/ch3/include/mpidpre.h | 7 +- src/mpid/ch3/include/mpidrma.h | 159 +++++++-------- src/mpid/ch3/src/ch3u_handle_op_req.c | 80 ++++++-- src/mpid/ch3/src/ch3u_request.c | 2 + src/mpid/ch3/src/ch3u_rma_ops.c | 30 --- src/mpid/ch3/src/ch3u_rma_progress.c | 218 +++++++++++---------- src/mpid/ch3/src/ch3u_rma_sync.c | 96 ++------- src/mpid/ch3/src/mpid_rma.c | 14 +- src/mpid/ch3/src/mpidi_rma.c | 7 +- 14 files changed, 459 insertions(+), 703 deletions(-) hooks/post-receive -- MPICH primary repository