[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.2b3-90-g82c2d65

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 82c2d6523528b56309beada7f76d28f866cd6ab1 (commit) via b997479938511c6a3e3b7323ab5864e537110e7e (commit) via bb3e74768a87562a983dbc25db23e87df4366505 (commit) via ec7850848485270924df41bf1c43e0bc83189901 (commit) via 4a53f4cca65af53857817acc11b4b34a2a19bc28 (commit) via bf114ec3827eeeb9ab17b26b5ebf7293575ce763 (commit) via 533b84c5adcab9fed07d3dab9abead0484298e1f (commit) via 22c16664e41a0f12eb984822909b8c452d066f63 (commit) via a59e611783f5fa0a6f3763d910ac0e1fe57f3350 (commit) via 56ede5541807579d6d4c4577a7cf21a3c12f1249 (commit) via 37cb95b14b693b634eb80fca442ff809acab10ba (commit) via 092d6c8ec115ca7fcbdd55f3879cbdc2dc8a969d (commit) via 69c2207df8e81625f0df1685a2242da93e5380cd (commit) via a7be0681b9ed8e4a49dec90f7de22574410027ed (commit) via 77c07868c0414781bae758ba8a7693287a1a24eb (commit) via 9fb9c2d36a51c55208036c3c4105dde8a004c84e (commit) via 0185437d948d81167a7aaf1e9dd02c4de7b891c8 (commit) via 5b1b3bbf6ef9b7c8d54e9fea202357e23d7cf243 (commit) via 408c4abf80cad55721d94917971fd85a860dbc5a (commit) via 74c2007fd55936f48a7d77d41a27b301ef263653 (commit) via 8a172d8b4cfb7489d54de9bb2fcafcbed98ff592 (commit) via 2db95ef98d98d968af42eeea9d0b9b2a4293bfcc (commit) via 299ede6e194bb0f7329511c83c3d0bf671ced860 (commit) via 35a05b3ca1e0cc60b1d1d607f408e0ab1e1f9ced (commit) via 98c791a6cfbc87698d4304ebdcb994b4c212fba1 (commit) via 6ee07af463b5de7fd11e9b94fc18ed5cd595a4dc (commit) via 6394c667ff22fed27bf0765c558e787a49ac66f8 (commit) via 86bb183473bb67c63a22c1b6e16d96d88581159f (commit) via 5a1dd95606ad83a8671a41ccc682076b9c19d63b (commit) via 1e321d463ba394c1b6fe9bc818c81be22e39f24c (commit) from 5bb037479a079a5e940a9c0e4e1f10f045cb9d7d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/82c2d6523528b56309beada7f76d28f866... commit 82c2d6523528b56309beada7f76d28f866cd6ab1 Author: Pavan Balaji <[email protected]> Date: Fri Jun 12 22:14:27 2015 -0500 White-space cleanup. diff --git a/src/mpid/ch3/include/mpid_rma_shm.h b/src/mpid/ch3/include/mpid_rma_shm.h index 1ad0351..f6d6b78 100644 --- a/src/mpid/ch3/include/mpid_rma_shm.h +++ b/src/mpid/ch3/include/mpid_rma_shm.h @@ -329,14 +329,15 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun if (shm_op) { MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); } - mpi_errno = do_accumulate_op((void*)origin_addr, origin_count, origin_datatype, - (void*)((char *)base+disp_unit*target_disp), target_count, target_datatype, - 0, op); + mpi_errno = do_accumulate_op((void *) origin_addr, origin_count, origin_datatype, + (void *) ((char *) base + disp_unit * target_disp), + target_count, target_datatype, 0, op); if (shm_op) { MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); } - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); goto fn_exit; } @@ -376,7 +377,8 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun packed_buf = MPIU_Malloc(stream_size); seg = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment"); + MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPID_Segment"); MPID_Segment_init(origin_addr, origin_count, origin_datatype, seg, 0); MPID_Segment_pack(seg, first, &last, packed_buf); MPID_Segment_free(seg); @@ -387,7 +389,8 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun void *tmpbuf = MPIU_Malloc(stream_count * predefined_dtp_extent); mpi_errno = MPIR_Localcopy(tmpbuf, stream_count, basic_type, packed_buf, stream_size, MPI_BYTE); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); MPIU_Free(packed_buf); packed_buf = tmpbuf; } @@ -396,15 +399,16 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); } - mpi_errno = do_accumulate_op((void*)packed_buf, stream_count, basic_type, - (void*)((char*)base+disp_unit*target_disp), target_count, target_datatype, - stream_offset, op); + mpi_errno = do_accumulate_op((void *) packed_buf, stream_count, basic_type, + (void *) ((char *) base + disp_unit * target_disp), + target_count, target_datatype, stream_offset, op); if (shm_op) { MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); } - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); MPIU_Free(packed_buf); } @@ -471,14 +475,15 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ if (is_empty_origin == TRUE || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { - mpi_errno = do_accumulate_op((void*)origin_addr, origin_count, origin_datatype, - (void*)((char *)base+disp_unit*target_disp), target_count, target_datatype, - 0, op); + mpi_errno = do_accumulate_op((void *) origin_addr, origin_count, origin_datatype, + (void *) ((char *) base + disp_unit * target_disp), + target_count, target_datatype, 0, op); if (shm_locked) { MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); } - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); goto fn_exit; } @@ -518,7 +523,8 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ packed_buf = MPIU_Malloc(stream_size); seg = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment"); + MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", + "MPID_Segment"); MPID_Segment_init(origin_addr, origin_count, origin_datatype, seg, 0); MPID_Segment_pack(seg, first, &last, packed_buf); MPID_Segment_free(seg); @@ -529,16 +535,18 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ void *tmpbuf = MPIU_Malloc(stream_count * predefined_dtp_extent); mpi_errno = MPIR_Localcopy(tmpbuf, stream_count, basic_type, packed_buf, stream_size, MPI_BYTE); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); MPIU_Free(packed_buf); packed_buf = tmpbuf; } - mpi_errno = do_accumulate_op((void*)packed_buf, stream_count, basic_type, - (void*)((char*)base+disp_unit*target_disp), target_count, target_datatype, - stream_offset, op); + mpi_errno = do_accumulate_op((void *) packed_buf, stream_count, basic_type, + (void *) ((char *) base + disp_unit * target_disp), + target_count, target_datatype, stream_offset, op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); MPIU_Free(packed_buf); } http://git.mpich.org/mpich.git/commitdiff/b997479938511c6a3e3b7323ab5864e537... commit b997479938511c6a3e3b7323ab5864e537110e7e Author: Xin Zhao <[email protected]> Date: Wed Jun 10 22:26:13 2015 -0500 Delete redundant function call. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 6611916..74731b0 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -38,7 +38,6 @@ static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able int *made_progress); static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, int *made_progress); -static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress); /* check if we can switch window-wide state: FENCE_ISSUED, PSCW_ISSUED, LOCK_ALL_ISSUED */ #undef FUNCNAME @@ -462,60 +461,6 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME issue_ops_win -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) -{ - int mpi_errno = MPI_SUCCESS; - int start_slot, end_slot, i, idx; - int is_able_to_issue = 0; - MPIDI_RMA_Target_t *target = NULL; - - (*made_progress) = 0; - - /* check and try to switch window state */ - mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, made_progress); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - if (!is_able_to_issue) - goto fn_exit; - - if (win_ptr->non_empty_slots == 0) - goto fn_exit; - - /* FIXME: we should optimize the issuing pattern here. */ - - start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots; - end_slot = start_slot + win_ptr->num_slots; - for (i = start_slot; i < end_slot; i++) { - if (i < win_ptr->num_slots) - idx = i; - else - idx = i - win_ptr->num_slots; - - target = win_ptr->slots[idx].target_list_head; - while (target != NULL) { - int temp_progress = 0; - - /* issue operations to this target */ - mpi_errno = issue_ops_target(win_ptr, target, &temp_progress); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - if (temp_progress) - (*made_progress) = 1; - - target = target->next; - } - } - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - #undef FUNCNAME #define FUNCNAME MPIDI_CH3I_RMA_Free_ops_before_completion @@ -823,17 +768,47 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress) { - int temp_progress = 0; int mpi_errno = MPI_SUCCESS; + int start_slot, end_slot, i, idx; + int is_able_to_issue = 0; + MPIDI_RMA_Target_t *target = NULL; (*made_progress) = 0; - /* issue operations on window */ - mpi_errno = issue_ops_win(win_ptr, &temp_progress); - if (mpi_errno) + /* check and try to switch window state */ + mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, made_progress); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (temp_progress) - (*made_progress) = 1; + if (!is_able_to_issue) + goto fn_exit; + + if (win_ptr->non_empty_slots == 0) + goto fn_exit; + + /* FIXME: we should optimize the issuing pattern here. */ + + start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots; + end_slot = start_slot + win_ptr->num_slots; + for (i = start_slot; i < end_slot; i++) { + if (i < win_ptr->num_slots) + idx = i; + else + idx = i - win_ptr->num_slots; + + target = win_ptr->slots[idx].target_list_head; + while (target != NULL) { + int temp_progress = 0; + + /* issue operations to this target */ + mpi_errno = issue_ops_target(win_ptr, target, &temp_progress); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); + if (temp_progress) + (*made_progress) = 1; + + target = target->next; + } + } fn_exit: return mpi_errno; http://git.mpich.org/mpich.git/commitdiff/bb3e74768a87562a983dbc25db23e87df4... commit bb3e74768a87562a983dbc25db23e87df4366505 Author: Xin Zhao <[email protected]> Date: Wed Jun 10 22:16:36 2015 -0500 Move check_and_switch_target/window_state() into issue_ops_target/window(). Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 787ffa5..6611916 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -298,10 +298,18 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ { MPIDI_RMA_Op_t *curr_op = NULL; MPIDI_CH3_Pkt_flags_t flags; + int is_able_to_issue = 0; int first_op = 1, mpi_errno = MPI_SUCCESS; (*made_progress) = 0; + /* check and try to switch target state */ + mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, made_progress); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); + if (!is_able_to_issue) + goto fn_exit; + if (win_ptr->non_empty_slots == 0 || target == NULL) goto fn_exit; @@ -462,10 +470,18 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) { int mpi_errno = MPI_SUCCESS; int start_slot, end_slot, i, idx; + int is_able_to_issue = 0; MPIDI_RMA_Target_t *target = NULL; (*made_progress) = 0; + /* check and try to switch window state */ + mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, made_progress); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); + if (!is_able_to_issue) + goto fn_exit; + if (win_ptr->non_empty_slots == 0) goto fn_exit; @@ -482,19 +498,6 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) target = win_ptr->slots[idx].target_list_head; while (target != NULL) { int temp_progress = 0; - int is_able_to_issue = 0; - - /* check target state */ - mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, - &temp_progress); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - if (temp_progress) - (*made_progress) = 1; - if (!is_able_to_issue) { - target = target->next; - continue; - } /* issue operations to this target */ mpi_errno = issue_ops_target(win_ptr, target, &temp_progress); @@ -800,15 +803,6 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - /* check target state */ - mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, &temp_progress); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - if (temp_progress) - (*made_progress) = 1; - if (!is_able_to_issue) - goto fn_exit; - /* issue operations to this target */ mpi_errno = issue_ops_target(win_ptr, target, &temp_progress); if (mpi_errno) @@ -830,20 +824,10 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress) { int temp_progress = 0; - int is_able_to_issue = 0; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; - /* check window state */ - mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - if (temp_progress) - (*made_progress) = 1; - if (!is_able_to_issue) - goto fn_exit; - /* issue operations on window */ mpi_errno = issue_ops_win(win_ptr, &temp_progress); if (mpi_errno) http://git.mpich.org/mpich.git/commitdiff/ec7850848485270924df41bf1c43e0bc83... commit ec7850848485270924df41bf1c43e0bc83189901 Author: Xin Zhao <[email protected]> Date: Fri May 29 10:58:40 2015 -0500 Return a flag from check_and_switch_target/window_state() on issuing ops. Here we make check_and_switch_target/window_state to return a flag indicating if the current window/target states are OK for issuing operations. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 8302c9a..787ffa5 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -33,8 +33,9 @@ cvars: */ static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, + int *is_able_to_issue, int *made_progress); +static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue, int *made_progress); -static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_progress); static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, int *made_progress); static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress); @@ -44,7 +45,8 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress); #define FUNCNAME check_and_switch_window_state #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_progress) +static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue, + int *made_progress) { MPID_Request *fence_req_ptr = NULL; int i, mpi_errno = MPI_SUCCESS; @@ -53,12 +55,15 @@ static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_pr MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE); (*made_progress) = 0; + (*is_able_to_issue) = 0; switch (win_ptr->states.access_state) { case MPIDI_RMA_FENCE_ISSUED: MPID_Request_get_ptr(win_ptr->fence_sync_req, fence_req_ptr); if (MPID_Request_is_complete(fence_req_ptr)) { win_ptr->states.access_state = MPIDI_RMA_FENCE_GRANTED; + (*is_able_to_issue) = 1; + MPID_Request_release(fence_req_ptr); win_ptr->fence_sync_req = MPI_REQUEST_NULL; @@ -74,6 +79,7 @@ static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_pr /* for MPI_MODE_NOCHECK and all targets on SHM, * we do not create PSCW requests on window. */ win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED; + (*is_able_to_issue) = 1; MPIDI_CH3I_num_active_issued_win--; MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0); @@ -97,6 +103,7 @@ static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_pr if (i == win_ptr->start_grp_size) { win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED; + (*is_able_to_issue) = 1; MPIDI_CH3I_num_active_issued_win--; MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0); @@ -112,10 +119,20 @@ static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_pr case MPIDI_RMA_LOCK_ALL_ISSUED: if (win_ptr->outstanding_locks == 0) { win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_GRANTED; + (*is_able_to_issue) = 1; + (*made_progress) = 1; } break; + case MPIDI_RMA_PER_TARGET: + case MPIDI_RMA_LOCK_ALL_CALLED: + case MPIDI_RMA_FENCE_GRANTED: + case MPIDI_RMA_PSCW_GRANTED: + case MPIDI_RMA_LOCK_ALL_GRANTED: + (*is_able_to_issue) = 1; + break; + default: break; } /* end of switch */ @@ -135,25 +152,17 @@ static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_pr #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, - int *made_progress) + int *is_able_to_issue, int *made_progress) { int rank = win_ptr->comm_ptr->rank; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; + (*is_able_to_issue) = 0; if (target == NULL) goto fn_exit; - /* This check should only be performed when window-wide sync is finished, or - * current sync is per-target sync. */ - if (win_ptr->states.access_state == MPIDI_RMA_NONE || - win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) { - goto fn_exit; - } - switch (target->access_state) { case MPIDI_RMA_LOCK_CALLED: if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE || @@ -269,6 +278,10 @@ static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Ta break; } /* end of switch */ + if (target->pending_op_list_head != NULL && target->access_state != MPIDI_RMA_LOCK_ISSUED) { + (*is_able_to_issue) = 1; + } + fn_exit: return mpi_errno; fn_fail: @@ -292,20 +305,6 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ if (win_ptr->non_empty_slots == 0 || target == NULL) goto fn_exit; - /* Exit if window-wide sync is not finished */ - if (win_ptr->states.access_state == MPIDI_RMA_NONE || - win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) - goto fn_exit; - - /* Exit if per-target sync is not finished */ - if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET || - win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) { - if (target->access_state == MPIDI_RMA_LOCK_ISSUED) - goto fn_exit; - } - /* Issue out operations in the list. */ curr_op = target->next_op_to_issue; while (curr_op != NULL) { @@ -470,13 +469,6 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) if (win_ptr->non_empty_slots == 0) goto fn_exit; - /* Exit if window-wide sync is not finished */ - if (win_ptr->states.access_state == MPIDI_RMA_NONE || - win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED || - win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) - goto fn_exit; - /* FIXME: we should optimize the issuing pattern here. */ start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots; @@ -490,13 +482,19 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) target = win_ptr->slots[idx].target_list_head; while (target != NULL) { int temp_progress = 0; + int is_able_to_issue = 0; /* check target state */ - mpi_errno = check_and_switch_target_state(win_ptr, target, &temp_progress); + mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, + &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) (*made_progress) = 1; + if (!is_able_to_issue) { + target = target->next; + continue; + } /* issue operations to this target */ mpi_errno = issue_ops_target(win_ptr, target, &temp_progress); @@ -782,17 +780,20 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int *made_progress) { int temp_progress = 0; + int is_able_to_issue = 0; MPIDI_RMA_Target_t *target = NULL; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; /* check window state */ - mpi_errno = check_and_switch_window_state(win_ptr, &temp_progress); + mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) (*made_progress) = 1; + if (!is_able_to_issue) + goto fn_exit; /* find target element */ mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, target_rank, &target); @@ -800,11 +801,13 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int MPIU_ERR_POP(mpi_errno); /* check target state */ - mpi_errno = check_and_switch_target_state(win_ptr, target, &temp_progress); + mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, &temp_progress); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (temp_progress) (*made_progress) = 1; + if (!is_able_to_issue) + goto fn_exit; /* issue operations to this target */ mpi_errno = issue_ops_target(win_ptr, target, &temp_progress); @@ -827,16 +830,19 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress) { int temp_progress = 0; + int is_able_to_issue = 0; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; /* check window state */ - mpi_errno = check_and_switch_window_state(win_ptr, &temp_progress); + mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) (*made_progress) = 1; + if (!is_able_to_issue) + goto fn_exit; /* issue operations on window */ mpi_errno = issue_ops_win(win_ptr, &temp_progress); http://git.mpich.org/mpich.git/commitdiff/4a53f4cca65af53857817acc11b4b34a2a... commit 4a53f4cca65af53857817acc11b4b34a2a19bc28 Author: Xin Zhao <[email protected]> Date: Fri May 29 10:23:42 2015 -0500 Change function name used in RMA progress. check_window_state ---> check_and_switch_window_state check_target_state ---> check_and_switch_target_state Both of those two functions are used to check and switch (if possible) RMA state. Here we change their name to proper ones. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index e106861..8302c9a 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -32,25 +32,25 @@ cvars: === END_MPI_T_CVAR_INFO_BLOCK === */ -static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, - int *made_progress); -static inline int check_window_state(MPID_Win * win_ptr, int *made_progress); +static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, + int *made_progress); +static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_progress); static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, int *made_progress); static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress); /* check if we can switch window-wide state: FENCE_ISSUED, PSCW_ISSUED, LOCK_ALL_ISSUED */ #undef FUNCNAME -#define FUNCNAME check_window_state +#define FUNCNAME check_and_switch_window_state #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int check_window_state(MPID_Win * win_ptr, int *made_progress) +static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *made_progress) { MPID_Request *fence_req_ptr = NULL; int i, mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_CHECK_WINDOW_STATE); + MPIDI_STATE_DECL(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE); - MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_WINDOW_STATE); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE); (*made_progress) = 0; @@ -121,7 +121,7 @@ static inline int check_window_state(MPID_Win * win_ptr, int *made_progress) } /* end of switch */ fn_exit: - MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_WINDOW_STATE); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: @@ -131,11 +131,11 @@ static inline int check_window_state(MPID_Win * win_ptr, int *made_progress) #undef FUNCNAME -#define FUNCNAME check_target_state +#define FUNCNAME check_and_switch_target_state #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, - int *made_progress) +static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, + int *made_progress) { int rank = win_ptr->comm_ptr->rank; int mpi_errno = MPI_SUCCESS; @@ -492,7 +492,7 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) int temp_progress = 0; /* check target state */ - mpi_errno = check_target_state(win_ptr, target, &temp_progress); + mpi_errno = check_and_switch_target_state(win_ptr, target, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) @@ -788,7 +788,7 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int (*made_progress) = 0; /* check window state */ - mpi_errno = check_window_state(win_ptr, &temp_progress); + mpi_errno = check_and_switch_window_state(win_ptr, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) @@ -800,7 +800,7 @@ int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int MPIU_ERR_POP(mpi_errno); /* check target state */ - mpi_errno = check_target_state(win_ptr, target, &temp_progress); + mpi_errno = check_and_switch_target_state(win_ptr, target, &temp_progress); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (temp_progress) @@ -832,7 +832,7 @@ int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress) (*made_progress) = 0; /* check window state */ - mpi_errno = check_window_state(win_ptr, &temp_progress); + mpi_errno = check_and_switch_window_state(win_ptr, &temp_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); if (temp_progress) http://git.mpich.org/mpich.git/commitdiff/bf114ec3827eeeb9ab17b26b5ebf729357... commit bf114ec3827eeeb9ab17b26b5ebf7293575ce763 Author: Xin Zhao <[email protected]> Date: Thu May 28 13:29:22 2015 -0500 Modify MPICH infrastructure to support atomic GET. When GACC/FOP is used with MPI_NO_OP, the operation is essentially an atomic GET. Originally MPICH implemented this by converting GACC/FOP to GET, which lost the atomicity of that operation. In this patch, we modify the implementation of GACC/FOP to support atomic GET. Main modifications are listed below: (1) When streaming GACC operation, originally we use origin data size to calculate the stream unit size. Since origin data is zero in atomic GET, here we use target data size instead to calculate the stream unit size. (2) On the origin side, if it is atomic GET, CH3 just issues packet header and metadata for derived datatypes (if needed) and does not try to issue from origin buffer; on the target side, after packet header and metadata for derived datatypes (if needed) are received, the final request handler is triggered, CH3 does not try to receive any data from origin. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 6eb300e..b13ed88 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -182,11 +182,21 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc, MPI_Datatype *datatypes = NULL; MPI_Aint dt_true_lb; MPIDI_CH3_Pkt_flags_t flags; + int is_empty_origin = FALSE; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_ISSUE_FROM_ORIGIN_BUFFER); MPIDI_FUNC_ENTER(MPID_STATE_ISSUE_FROM_ORIGIN_BUFFER); + /* Judge if origin buffer is empty (this can only happens for + * GACC and FOP when op is MPI_NO_OP). */ + if ((rma_op->pkt).type == MPIDI_CH3_PKT_GET_ACCUM || (rma_op->pkt).type == MPIDI_CH3_PKT_FOP) { + MPI_Op op; + MPIDI_CH3_PKT_RMA_GET_OP(rma_op->pkt, op, mpi_errno); + if (op == MPI_NO_OP) + is_empty_origin = TRUE; + } + /* Judge if target datatype is derived datatype. */ MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(rma_op->pkt, target_datatype, mpi_errno); if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { @@ -203,14 +213,21 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc, } } - /* Judge if origin datatype is derived datatype. */ - if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) { - MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp); - } + if (is_empty_origin == FALSE) { + /* Judge if origin datatype is derived datatype. */ + if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) { + MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp); + } - /* check if origin data is contiguous and get true lb */ - MPID_Datatype_is_contig(rma_op->origin_datatype, &is_origin_contig); - MPID_Datatype_get_true_lb(rma_op->origin_datatype, &dt_true_lb); + /* check if origin data is contiguous and get true lb */ + MPID_Datatype_is_contig(rma_op->origin_datatype, &is_origin_contig); + MPID_Datatype_get_true_lb(rma_op->origin_datatype, &dt_true_lb); + } + else { + /* origin buffer is empty, mark origin data as contig and true_lb as 0. */ + is_origin_contig = 1; + dt_true_lb = 0; + } iov[iovcnt].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & (rma_op->pkt); iov[iovcnt].MPID_IOV_LEN = sizeof(rma_op->pkt); @@ -225,10 +242,12 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc, * (3) origin datatype is contiguous (do not need to pack the data and send); */ - iov[iovcnt].MPID_IOV_BUF = - (MPID_IOV_BUF_CAST) ((char *) rma_op->origin_addr + dt_true_lb + stream_offset); - iov[iovcnt].MPID_IOV_LEN = stream_size; - iovcnt++; + if (is_empty_origin == FALSE) { + iov[iovcnt].MPID_IOV_BUF = + (MPID_IOV_BUF_CAST) ((char *) rma_op->origin_addr + dt_true_lb + stream_offset); + iov[iovcnt].MPID_IOV_LEN = stream_size; + iovcnt++; + } MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, &req); @@ -294,10 +313,13 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc, if (is_origin_contig) { /* origin data is contiguous */ - iov[iovcnt].MPID_IOV_BUF = - (MPID_IOV_BUF_CAST) ((char *) rma_op->origin_addr + dt_true_lb + stream_offset); - iov[iovcnt].MPID_IOV_LEN = stream_size; - iovcnt++; + + if (is_empty_origin == FALSE) { + iov[iovcnt].MPID_IOV_BUF = + (MPID_IOV_BUF_CAST) ((char *) rma_op->origin_addr + dt_true_lb + stream_offset); + iov[iovcnt].MPID_IOV_LEN = stream_size; + iovcnt++; + } MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, iovcnt); @@ -336,7 +358,24 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc, /* create a new datatype containing the dtype_info, dataloop, and origin data */ - if (flags & MPIDI_CH3_PKT_FLAG_RMA_STREAM) { + if (is_empty_origin == TRUE) { + count = 2; + ints = (int *) MPIU_Malloc(sizeof(int) * (count + 1)); + blocklens = &ints[1]; + displaces = (MPI_Aint *) MPIU_Malloc(sizeof(MPI_Aint) * count); + datatypes = (MPI_Datatype *) MPIU_Malloc(sizeof(MPI_Datatype) * count); + + ints[0] = count; + + displaces[0] = MPIU_PtrToAint(&(rma_op->dtype_info)); + blocklens[0] = sizeof(MPIDI_RMA_dtype_info); + datatypes[0] = MPI_BYTE; + + displaces[1] = MPIU_PtrToAint(rma_op->dataloop); + MPIU_Assign_trunc(blocklens[1], target_dtp->dataloop_size, int); + datatypes[1] = MPI_BYTE; + } + else if (flags & MPIDI_CH3_PKT_FLAG_RMA_STREAM) { segp = MPID_Segment_alloc(); MPIU_ERR_CHKANDJUMP1(segp == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc"); @@ -685,8 +724,8 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPI_Aint stream_elem_count, stream_unit_count; MPI_Aint predefined_dtp_size, predefined_dtp_count, predefined_dtp_extent; MPI_Aint total_len, rest_len; - MPI_Aint origin_dtp_size; - MPID_Datatype *origin_dtp_ptr = NULL; + MPI_Aint target_dtp_size; + MPID_Datatype *target_dtp_ptr = NULL; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_ISSUE_GET_ACC_OP); @@ -746,22 +785,22 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, goto fn_exit; } - /* Get total length of origin data */ - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_dtp_size); - total_len = origin_dtp_size * rma_op->origin_count; + /* Get total length of target data */ + MPID_Datatype_get_size_macro(get_accum_pkt->datatype, target_dtp_size); + total_len = target_dtp_size * get_accum_pkt->count; /* Get size and count for predefined datatype elements */ - if (MPIR_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype)) { - predefined_dtp_size = origin_dtp_size; - predefined_dtp_count = rma_op->origin_count; - MPID_Datatype_get_extent_macro(rma_op->origin_datatype, predefined_dtp_extent); + if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { + predefined_dtp_size = target_dtp_size; + predefined_dtp_count = get_accum_pkt->count; + MPID_Datatype_get_extent_macro(get_accum_pkt->datatype, predefined_dtp_extent); } else { - MPID_Datatype_get_ptr(rma_op->origin_datatype, origin_dtp_ptr); - MPIU_Assert(origin_dtp_ptr != NULL && origin_dtp_ptr->basic_type != MPI_DATATYPE_NULL); - MPID_Datatype_get_size_macro(origin_dtp_ptr->basic_type, predefined_dtp_size); + MPID_Datatype_get_ptr(get_accum_pkt->datatype, target_dtp_ptr); + MPIU_Assert(target_dtp_ptr != NULL && target_dtp_ptr->basic_type != MPI_DATATYPE_NULL); + MPID_Datatype_get_size_macro(target_dtp_ptr->basic_type, predefined_dtp_size); predefined_dtp_count = total_len / predefined_dtp_size; - MPID_Datatype_get_extent_macro(origin_dtp_ptr->basic_type, predefined_dtp_extent); + MPID_Datatype_get_extent_macro(target_dtp_ptr->basic_type, predefined_dtp_extent); } MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 && predefined_dtp_extent > 0); diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 1abc4b2..046ca96 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -367,6 +367,29 @@ MPIDI_CH3_PKT_DEFS } \ } +#define MPIDI_CH3_PKT_RMA_GET_OP(pkt_, op_, err_) \ + { \ + /* This macro returns op in RMA operation packets (ACC, GACC, \ + FOP) */ \ + err_ = MPI_SUCCESS; \ + switch((pkt_).type) { \ + case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ + op_ = (pkt_).accum.op; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ + op_ = (pkt_).get_accum.op; \ + break; \ + case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ + op_ = (pkt_).fop.op; \ + break; \ + default: \ + MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ + } \ + } + #define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_) \ { \ /* This macro erases flags in RMA operation packets (PUT, GET, \ diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index db1e7ec..bfaa7a7 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -357,7 +357,7 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, MPI_Aint type_size = 0; MPI_Aint type_extent; MPIDI_msg_sz_t recv_data_sz = 0; - MPIDI_msg_sz_t buf_size; + MPIDI_msg_sz_t buf_size = 0; MPID_Request *req = NULL; MPI_Datatype target_dtp; int target_count; @@ -382,11 +382,15 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, else { MPI_Aint stream_elem_count; MPI_Aint total_len; - - stream_elem_count = MPIDI_CH3U_SRBuf_size / type_extent; - total_len = type_size * target_count; - recv_data_sz = MPIR_MIN(total_len, type_size * stream_elem_count); - buf_size = type_extent * (recv_data_sz / type_size); + MPI_Op op; + + MPIDI_CH3_PKT_RMA_GET_OP((*pkt), op, mpi_errno); + if (op != MPI_NO_OP) { + stream_elem_count = MPIDI_CH3U_SRBuf_size / type_extent; + total_len = type_size * target_count; + recv_data_sz = MPIR_MIN(total_len, type_size * stream_elem_count); + buf_size = type_extent * (recv_data_sz / type_size); + } } if (flags & MPIDI_CH3_PKT_FLAG_RMA_STREAM) { @@ -465,7 +469,7 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - MPIU_Assert(req->dev.recv_data_sz > 0); + MPIU_Assert(req->dev.recv_data_sz >= 0); } else { req->dev.user_buf = new_ptr->data; @@ -478,7 +482,7 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc, data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - MPIU_Assert(req->dev.recv_data_sz > 0); + MPIU_Assert(req->dev.recv_data_sz >= 0); } mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index b2c6e29..2eddc41 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -242,11 +242,19 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq MPI_Aint predef_count, predef_dtp_size; MPI_Aint dt_true_lb; MPI_Aint stream_offset; + int is_empty_origin = FALSE; + MPI_Aint extent, type_size; + MPI_Aint stream_data_len, total_len; MPIU_CHKPMEM_DECL(1); MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE); + /* Judge if origin buffer is empty */ + if (rreq->dev.op == MPI_NO_OP) { + is_empty_origin = TRUE; + } + MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) @@ -256,8 +264,14 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq } MPIU_Assert(basic_type != MPI_DATATYPE_NULL); + /* Use target data to calculate current stream unit size */ + MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size); + total_len = type_size * rreq->dev.user_count; + MPID_Datatype_get_extent_macro(rreq->dev.datatype, extent); + stream_data_len = MPIR_MIN(total_len, (MPIDI_CH3U_SRBuf_size / extent) * type_size); + MPID_Datatype_get_size_macro(basic_type, predef_dtp_size); - predef_count = rreq->dev.recv_data_sz / predef_dtp_size; + predef_count = stream_data_len / predef_dtp_size; MPIU_Assert(predef_count > 0); MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); @@ -288,7 +302,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq MPIU_Object_set_ref(resp_req, 1); MPIDI_Request_set_type(resp_req, MPIDI_REQUEST_TYPE_GET_ACCUM_RESP); - MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, rreq->dev.recv_data_sz, + MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, stream_data_len, mpi_errno, "GACC resp. buffer"); /* NOTE: 'copy data + ACC' needs to be atomic */ @@ -301,12 +315,12 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq if (is_contig) { MPIU_Memcpy(resp_req->dev.user_buf, (void *) ((char *) rreq->dev.real_user_buf + dt_true_lb + - stream_offset), rreq->dev.recv_data_sz); + stream_offset), stream_data_len); } else { MPID_Segment *seg = MPID_Segment_alloc(); MPI_Aint first = stream_offset; - MPI_Aint last = first + rreq->dev.recv_data_sz; + MPI_Aint last = first + stream_data_len; if (seg == NULL) { if (win_ptr->shm_allocated == TRUE) @@ -343,7 +357,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) resp_req->dev.user_buf); - iov[1].MPID_IOV_LEN = rreq->dev.recv_data_sz; + iov[1].MPID_IOV_LEN = stream_data_len; iovcnt = 2; MPIU_THREAD_CS_ENTER(CH3COMM, vc); @@ -357,8 +371,12 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_GET_ACCUM_RECV); - /* free the temporary buffer */ - MPIDI_CH3U_SRBuf_free(rreq); + if (is_empty_origin == FALSE) { + /* free the temporary buffer. + * When origin data is zero, there + * is no temporary buffer allocated */ + MPIDI_CH3U_SRBuf_free(rreq); + } /* mark data transfer as complete and decrement CC */ MPIDI_CH3U_Request_complete(rreq); @@ -391,11 +409,17 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; int is_contig; + int is_empty_origin = FALSE; MPIU_CHKPMEM_DECL(1); MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + /* Judge if origin buffer is empty */ + if (rreq->dev.op == MPI_NO_OP) { + is_empty_origin = TRUE; + } + MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_FOP_RECV); MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); @@ -478,8 +502,12 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); - /* free the temporary buffer */ - MPIU_Free((char *) rreq->dev.user_buf); + if (is_empty_origin == FALSE) { + /* free the temporary buffer. + * When origin data is zero, there + * is no temporary buffer allocated */ + MPIU_Free((char *) rreq->dev.user_buf); + } /* mark data transfer as complete and decrement CC */ MPIDI_CH3U_Request_complete(rreq); @@ -650,7 +678,7 @@ int MPIDI_CH3_ReqHandler_AccumMetadataRecvComplete(MPIDI_VC_t * vc ATTRIBUTE((un #define FUNCNAME MPIDI_CH3_ReqHandler_GaccumMetadataRecvComplete #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPIDI_CH3_ReqHandler_GaccumMetadataRecvComplete(MPIDI_VC_t * vc ATTRIBUTE((unused)), +int MPIDI_CH3_ReqHandler_GaccumMetadataRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, int *complete) { int mpi_errno = MPI_SUCCESS; @@ -660,10 +688,16 @@ int MPIDI_CH3_ReqHandler_GaccumMetadataRecvComplete(MPIDI_VC_t * vc ATTRIBUTE((u MPI_Aint stream_offset; MPI_Aint type_size; MPI_Datatype basic_dtp; + int is_empty_origin = FALSE; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMMETADATARECVCOMPLETE); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMMETADATARECVCOMPLETE); + /* Judge if origin buffer is empty */ + if (rreq->dev.op == MPI_NO_OP) { + is_empty_origin = TRUE; + } + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_STREAM) { MPIU_Assert(rreq->dev.ext_hdr_ptr != NULL); stream_offset = ((MPIDI_CH3_Ext_pkt_get_accum_t *) rreq->dev.ext_hdr_ptr)->stream_offset; @@ -698,49 +732,60 @@ int MPIDI_CH3_ReqHandler_GaccumMetadataRecvComplete(MPIDI_VC_t * vc ATTRIBUTE((u basic_dtp = rreq->dev.datatype; } - MPID_Datatype_get_size_macro(basic_dtp, basic_type_size); - MPID_Datatype_get_extent_macro(basic_dtp, basic_type_extent); + if (is_empty_origin == TRUE) { + rreq->dev.recv_data_sz = 0; - MPIU_Assert(!MPIDI_Request_get_srbuf_flag(rreq)); - /* allocate a SRBuf for receiving stream unit */ - MPIDI_CH3U_SRBuf_alloc(rreq, MPIDI_CH3U_SRBuf_size); - /* --BEGIN ERROR HANDLING-- */ - if (rreq->dev.tmpbuf_sz == 0) { - MPIU_DBG_MSG(CH3_CHANNEL, TYPICAL, "SRBuf allocation failure"); - mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, - FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", - "**nomem %d", MPIDI_CH3U_SRBuf_size); - rreq->status.MPI_ERROR = mpi_errno; - goto fn_fail; + /* There is no origin data coming, directly call final req handler. */ + mpi_errno = MPIDI_CH3_ReqHandler_GaccumRecvComplete(vc, rreq, complete); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); } - /* --END ERROR HANDLING-- */ + else { + MPID_Datatype_get_size_macro(basic_dtp, basic_type_size); + MPID_Datatype_get_extent_macro(basic_dtp, basic_type_extent); + + MPIU_Assert(!MPIDI_Request_get_srbuf_flag(rreq)); + /* allocate a SRBuf for receiving stream unit */ + MPIDI_CH3U_SRBuf_alloc(rreq, MPIDI_CH3U_SRBuf_size); + /* --BEGIN ERROR HANDLING-- */ + if (rreq->dev.tmpbuf_sz == 0) { + MPIU_DBG_MSG(CH3_CHANNEL, TYPICAL, "SRBuf allocation failure"); + mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, + FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", + "**nomem %d", MPIDI_CH3U_SRBuf_size); + rreq->status.MPI_ERROR = mpi_errno; + goto fn_fail; + } + /* --END ERROR HANDLING-- */ - rreq->dev.user_buf = rreq->dev.tmpbuf; + rreq->dev.user_buf = rreq->dev.tmpbuf; - total_len = type_size * rreq->dev.user_count; - rest_len = total_len - stream_offset; - stream_elem_count = MPIDI_CH3U_SRBuf_size / basic_type_extent; + total_len = type_size * rreq->dev.user_count; + rest_len = total_len - stream_offset; + stream_elem_count = MPIDI_CH3U_SRBuf_size / basic_type_extent; - rreq->dev.recv_data_sz = MPIR_MIN(rest_len, stream_elem_count * basic_type_size); + rreq->dev.recv_data_sz = MPIR_MIN(rest_len, stream_elem_count * basic_type_size); - rreq->dev.segment_ptr = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", - "**nomem %s", "MPID_Segment_alloc"); + rreq->dev.segment_ptr = MPID_Segment_alloc(); + MPIU_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", + "**nomem %s", "MPID_Segment_alloc"); - MPID_Segment_init(rreq->dev.user_buf, - (rreq->dev.recv_data_sz / basic_type_size), - basic_dtp, rreq->dev.segment_ptr, 0); - rreq->dev.segment_first = 0; - rreq->dev.segment_size = rreq->dev.recv_data_sz; + MPID_Segment_init(rreq->dev.user_buf, + (rreq->dev.recv_data_sz / basic_type_size), + basic_dtp, rreq->dev.segment_ptr, 0); + rreq->dev.segment_first = 0; + rreq->dev.segment_size = rreq->dev.recv_data_sz; - mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|loadrecviov"); + mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|loadrecviov"); + } + if (!rreq->dev.OnDataAvail) + rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumRecvComplete; + + *complete = FALSE; } - if (!rreq->dev.OnDataAvail) - rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumRecvComplete; - *complete = FALSE; fn_fail: MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMMETADATARECVCOMPLETE); return mpi_errno; diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index f0de4d6..39f9f57 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -708,11 +708,17 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; MPI_Aint origin_type_size; + MPI_Aint target_type_size; int use_immed_pkt = FALSE, i; int is_origin_contig, is_target_contig, is_result_contig; MPI_Aint stream_elem_count, stream_unit_count; MPI_Aint predefined_dtp_size, predefined_dtp_count, predefined_dtp_extent; MPID_Datatype *origin_dtp = NULL, *target_dtp = NULL, *result_dtp = NULL; + int is_empty_origin = FALSE; + + /* Judge if origin buffer is empty */ + if (op == MPI_NO_OP) + is_empty_origin = TRUE; /* Append the operation to the window's RMA ops queue */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); @@ -738,7 +744,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /* if source or target datatypes are derived, increment their * reference counts */ - if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { + if (is_empty_origin == FALSE && !MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, origin_dtp); } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { @@ -748,20 +754,28 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_get_ptr(target_datatype, target_dtp); } - MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); - MPIU_Assign_trunc(orig_data_sz, origin_count * origin_type_size, MPIDI_msg_sz_t); + if (is_empty_origin == FALSE) { + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(orig_data_sz, origin_count * origin_type_size, MPIDI_msg_sz_t); + } + else { + /* If origin buffer is empty, set origin data size to 0 */ + orig_data_sz = 0; + } + + MPID_Datatype_get_size_macro(target_datatype, target_type_size); /* Get size and count for predefined datatype elements */ - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { - predefined_dtp_size = origin_type_size; - predefined_dtp_count = origin_count; - MPID_Datatype_get_extent_macro(origin_datatype, predefined_dtp_extent); + if (MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { + predefined_dtp_size = target_type_size; + predefined_dtp_count = target_count; + MPID_Datatype_get_extent_macro(target_datatype, predefined_dtp_extent); } else { - MPIU_Assert(origin_dtp->basic_type != MPI_DATATYPE_NULL); - MPID_Datatype_get_size_macro(origin_dtp->basic_type, predefined_dtp_size); - predefined_dtp_count = orig_data_sz / predefined_dtp_size; - MPID_Datatype_get_extent_macro(origin_dtp->basic_type, predefined_dtp_extent); + MPIU_Assert(target_dtp->basic_type != MPI_DATATYPE_NULL); + MPID_Datatype_get_size_macro(target_dtp->basic_type, predefined_dtp_size); + predefined_dtp_count = target_data_sz / predefined_dtp_size; + MPID_Datatype_get_extent_macro(target_dtp->basic_type, predefined_dtp_extent); } MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 && predefined_dtp_extent > 0); @@ -784,21 +798,27 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, } } - MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); + if (is_empty_origin == FALSE) { + MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); + } + else { + /* If origin buffer is empty, mark origin data as contig data */ + is_origin_contig = 1; + } MPID_Datatype_is_contig(target_datatype, &is_target_contig); MPID_Datatype_is_contig(result_datatype, &is_result_contig); /* Judge if we can use IMMED data packet */ - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + if ((is_empty_origin == TRUE || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) && MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig && is_result_contig) { - if (orig_data_sz <= MPIDI_RMA_IMMED_BYTES) + if (target_data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } /* Judge if this operation is a piggyback candidate */ - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + if ((is_empty_origin == TRUE || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) && MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 29b00ba..8dbabd1 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -979,39 +979,53 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); } else { + int is_empty_origin = FALSE; + + /* Judge if origin data is zero. */ + if (get_accum_pkt->op == MPI_NO_OP) + is_empty_origin = TRUE; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumRecvComplete; - MPID_Datatype_get_extent_macro(get_accum_pkt->datatype, extent); + if (is_empty_origin == TRUE) { + req->dev.recv_data_sz = 0; - MPIU_Assert(!MPIDI_Request_get_srbuf_flag(req)); - /* allocate a SRBuf for receiving stream unit */ - MPIDI_CH3U_SRBuf_alloc(req, MPIDI_CH3U_SRBuf_size); - /* --BEGIN ERROR HANDLING-- */ - if (req->dev.tmpbuf_sz == 0) { - MPIU_DBG_MSG(CH3_CHANNEL, TYPICAL, "SRBuf allocation failure"); - mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, - FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", - "**nomem %d", MPIDI_CH3U_SRBuf_size); - req->status.MPI_ERROR = mpi_errno; - goto fn_fail; + *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; } - /* --END ERROR HANDLING-- */ + else { + MPID_Datatype_get_extent_macro(get_accum_pkt->datatype, extent); + + MPIU_Assert(!MPIDI_Request_get_srbuf_flag(req)); + /* allocate a SRBuf for receiving stream unit */ + MPIDI_CH3U_SRBuf_alloc(req, MPIDI_CH3U_SRBuf_size); + /* --BEGIN ERROR HANDLING-- */ + if (req->dev.tmpbuf_sz == 0) { + MPIU_DBG_MSG(CH3_CHANNEL, TYPICAL, "SRBuf allocation failure"); + mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, + FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", + "**nomem %d", MPIDI_CH3U_SRBuf_size); + req->status.MPI_ERROR = mpi_errno; + goto fn_fail; + } + /* --END ERROR HANDLING-- */ - req->dev.user_buf = req->dev.tmpbuf; + req->dev.user_buf = req->dev.tmpbuf; - MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); - total_len = type_size * get_accum_pkt->count; - stream_elem_count = MPIDI_CH3U_SRBuf_size / extent; + MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); + total_len = type_size * get_accum_pkt->count; + stream_elem_count = MPIDI_CH3U_SRBuf_size / extent; - req->dev.recv_data_sz = MPIR_MIN(total_len, stream_elem_count * type_size); - MPIU_Assert(req->dev.recv_data_sz > 0); + req->dev.recv_data_sz = MPIR_MIN(total_len, stream_elem_count * type_size); + MPIU_Assert(req->dev.recv_data_sz > 0); - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + } if (complete) { mpi_errno = MPIDI_CH3_ReqHandler_GaccumRecvComplete(vc, req, &complete); @@ -1404,6 +1418,11 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t data_len; MPI_Aint extent; int complete = 0; + int is_empty_origin = FALSE; + + /* Judge if origin data is zero. */ + if (fop_pkt->op == MPI_NO_OP) + is_empty_origin = TRUE; req = MPID_Request_create(); MPIU_Object_set_ref(req, 1); @@ -1420,26 +1439,34 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.datatype = fop_pkt->datatype; req->dev.user_count = 1; - /* get start location of data and length of data */ - data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); - data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - - MPID_Datatype_get_extent_macro(fop_pkt->datatype, extent); + if (is_empty_origin == TRUE) { + req->dev.recv_data_sz = 0; - req->dev.user_buf = MPIU_Malloc(extent); - if (!req->dev.user_buf) { - MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", extent); + *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; } + else { + /* get start location of data and length of data */ + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - req->dev.recv_data_sz = type_size; - MPIU_Assert(req->dev.recv_data_sz > 0); + MPID_Datatype_get_extent_macro(fop_pkt->datatype, extent); - mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + req->dev.user_buf = MPIU_Malloc(extent); + if (!req->dev.user_buf) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", extent); + } - /* return the number of bytes processed in this function */ - *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + req->dev.recv_data_sz = type_size; + MPIU_Assert(req->dev.recv_data_sz > 0); + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + } if (complete) { mpi_errno = MPIDI_CH3_ReqHandler_FOPRecvComplete(vc, req, &complete); http://git.mpich.org/mpich.git/commitdiff/533b84c5adcab9fed07d3dab9abead0484... commit 533b84c5adcab9fed07d3dab9abead0484298e1f Author: Xin Zhao <[email protected]> Date: Thu May 28 13:15:04 2015 -0500 Delete code of converting GACC/FOP to GET when op is MPI_NO_OP. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 3d5568e..f0de4d6 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -706,6 +706,13 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, } else { MPIDI_RMA_Op_t *op_ptr = NULL; + MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; + MPI_Aint origin_type_size; + int use_immed_pkt = FALSE, i; + int is_origin_contig, is_target_contig, is_result_contig; + MPI_Aint stream_elem_count, stream_unit_count; + MPI_Aint predefined_dtp_size, predefined_dtp_count, predefined_dtp_extent; + MPID_Datatype *origin_dtp = NULL, *target_dtp = NULL, *result_dtp = NULL; /* Append the operation to the window's RMA ops queue */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); @@ -716,186 +723,115 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - if (op == MPI_NO_OP) { - /* Convert GAcc to a Get */ - MPIDI_CH3_Pkt_get_t *get_pkt; - int use_immed_resp_pkt = FALSE; - int is_result_contig, is_target_contig; - - /******************** Setting operation struct areas ***********************/ - - op_ptr->origin_addr = result_addr; - op_ptr->origin_count = result_count; - op_ptr->origin_datatype = result_datatype; - op_ptr->target_rank = target_rank; - - /* Remember user request */ - op_ptr->ureq = ureq; + /******************** Setting operation struct areas ***********************/ - if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { - MPID_Datatype_get_ptr(result_datatype, dtp); - MPID_Datatype_add_ref(dtp); - } - if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - MPID_Datatype_get_ptr(target_datatype, dtp); - MPID_Datatype_add_ref(dtp); - } + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = origin_count; + op_ptr->origin_datatype = origin_datatype; + op_ptr->result_addr = result_addr; + op_ptr->result_count = result_count; + op_ptr->result_datatype = result_datatype; + op_ptr->target_rank = target_rank; - MPID_Datatype_is_contig(result_datatype, &is_result_contig); - MPID_Datatype_is_contig(target_datatype, &is_target_contig); + /* Remember user request */ + op_ptr->ureq = ureq; - /* Judge if we can use IMMED data response packet */ - if (MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && - is_result_contig && is_target_contig) { - if (target_data_sz <= MPIDI_RMA_IMMED_BYTES) - use_immed_resp_pkt = TRUE; - } + /* if source or target datatypes are derived, increment their + * reference counts */ + if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { + MPID_Datatype_get_ptr(origin_datatype, origin_dtp); + } + if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { + MPID_Datatype_get_ptr(result_datatype, result_dtp); + } + if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { + MPID_Datatype_get_ptr(target_datatype, target_dtp); + } - /* Judge if this operation is a piggyback candidate */ - if (MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes - * for both origin and target data. We should extend this optimization to derived - * datatypes as well. */ - op_ptr->piggyback_lock_candidate = 1; - } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(orig_data_sz, origin_count * origin_type_size, MPIDI_msg_sz_t); - /************** Setting packet struct areas in operation ****************/ - - get_pkt = &(op_ptr->pkt.get); - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + - win_ptr->basic_info_table[target_rank].disp_unit * target_disp; - get_pkt->count = target_count; - get_pkt->datatype = target_datatype; - get_pkt->info.dataloop_size = 0; - get_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; - get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (use_immed_resp_pkt == TRUE) - get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; + /* Get size and count for predefined datatype elements */ + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { + predefined_dtp_size = origin_type_size; + predefined_dtp_count = origin_count; + MPID_Datatype_get_extent_macro(origin_datatype, predefined_dtp_extent); } - else { - MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; - MPI_Aint origin_type_size; - int use_immed_pkt = FALSE; - int is_origin_contig, is_target_contig, is_result_contig; - MPI_Aint stream_elem_count, stream_unit_count; - MPI_Aint predefined_dtp_size, predefined_dtp_count, predefined_dtp_extent; - MPID_Datatype *origin_dtp = NULL, *target_dtp = NULL, *result_dtp = NULL; - int i; - - /******************** Setting operation struct areas ***********************/ - - op_ptr->origin_addr = (void *) origin_addr; - op_ptr->origin_count = origin_count; - op_ptr->origin_datatype = origin_datatype; - op_ptr->result_addr = result_addr; - op_ptr->result_count = result_count; - op_ptr->result_datatype = result_datatype; - op_ptr->target_rank = target_rank; - - /* Remember user request */ - op_ptr->ureq = ureq; - - /* if source or target datatypes are derived, increment their - * reference counts */ - if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { - MPID_Datatype_get_ptr(origin_datatype, origin_dtp); - } - if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { - MPID_Datatype_get_ptr(result_datatype, result_dtp); - } - if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - MPID_Datatype_get_ptr(target_datatype, target_dtp); - } + MPIU_Assert(origin_dtp->basic_type != MPI_DATATYPE_NULL); + MPID_Datatype_get_size_macro(origin_dtp->basic_type, predefined_dtp_size); + predefined_dtp_count = orig_data_sz / predefined_dtp_size; + MPID_Datatype_get_extent_macro(origin_dtp->basic_type, predefined_dtp_extent); + } + MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 && + predefined_dtp_extent > 0); - MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); - MPIU_Assign_trunc(orig_data_sz, origin_count * origin_type_size, MPIDI_msg_sz_t); + /* Calculate number of predefined elements in each stream unit, and + * total number of stream units. */ + stream_elem_count = MPIDI_CH3U_Acc_stream_size / predefined_dtp_extent; + stream_unit_count = (predefined_dtp_count - 1) / stream_elem_count + 1; + MPIU_Assert(stream_elem_count > 0 && stream_unit_count > 0); - /* Get size and count for predefined datatype elements */ - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { - predefined_dtp_size = origin_type_size; - predefined_dtp_count = origin_count; - MPID_Datatype_get_extent_macro(origin_datatype, predefined_dtp_extent); + for (i = 0; i < stream_unit_count; i++) { + if (origin_dtp != NULL) { + MPID_Datatype_add_ref(origin_dtp); } - else { - MPIU_Assert(origin_dtp->basic_type != MPI_DATATYPE_NULL); - MPID_Datatype_get_size_macro(origin_dtp->basic_type, predefined_dtp_size); - predefined_dtp_count = orig_data_sz / predefined_dtp_size; - MPID_Datatype_get_extent_macro(origin_dtp->basic_type, predefined_dtp_extent); + if (target_dtp != NULL) { + MPID_Datatype_add_ref(target_dtp); } - MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 && - predefined_dtp_extent > 0); - - /* Calculate number of predefined elements in each stream unit, and - * total number of stream units. */ - stream_elem_count = MPIDI_CH3U_Acc_stream_size / predefined_dtp_extent; - stream_unit_count = (predefined_dtp_count - 1) / stream_elem_count + 1; - MPIU_Assert(stream_elem_count > 0 && stream_unit_count > 0); - - for (i = 0; i < stream_unit_count; i++) { - if (origin_dtp != NULL) { - MPID_Datatype_add_ref(origin_dtp); - } - if (target_dtp != NULL) { - MPID_Datatype_add_ref(target_dtp); - } - if (result_dtp != NULL) { - MPID_Datatype_add_ref(result_dtp); - } + if (result_dtp != NULL) { + MPID_Datatype_add_ref(result_dtp); } + } - MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); - MPID_Datatype_is_contig(target_datatype, &is_target_contig); - MPID_Datatype_is_contig(result_datatype, &is_result_contig); + MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); + MPID_Datatype_is_contig(target_datatype, &is_target_contig); + MPID_Datatype_is_contig(result_datatype, &is_result_contig); - /* Judge if we can use IMMED data packet */ - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && - is_origin_contig && is_target_contig && is_result_contig) { - if (orig_data_sz <= MPIDI_RMA_IMMED_BYTES) - use_immed_pkt = TRUE; - } + /* Judge if we can use IMMED data packet */ + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && + is_origin_contig && is_target_contig && is_result_contig) { + if (orig_data_sz <= MPIDI_RMA_IMMED_BYTES) + use_immed_pkt = TRUE; + } - /* Judge if this operation is a piggyback candidate */ - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && - MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes - * for origin, target and result data. We should extend this optimization to derived - * datatypes as well. */ - if (orig_data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) - op_ptr->piggyback_lock_candidate = 1; - } + /* Judge if this operation is a piggyback candidate */ + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { + /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes + * for origin, target and result data. We should extend this optimization to derived + * datatypes as well. */ + if (orig_data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) + op_ptr->piggyback_lock_candidate = 1; + } - /************** Setting packet struct areas in operation ****************/ + /************** Setting packet struct areas in operation ****************/ - get_accum_pkt = &(op_ptr->pkt.get_accum); + get_accum_pkt = &(op_ptr->pkt.get_accum); - if (use_immed_pkt) { - MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED); - } - else { - MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); - } + if (use_immed_pkt) { + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED); + } + else { + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + } - get_accum_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + - win_ptr->basic_info_table[target_rank].disp_unit * target_disp; - get_accum_pkt->count = target_count; - get_accum_pkt->datatype = target_datatype; - get_accum_pkt->info.dataloop_size = 0; - get_accum_pkt->op = op; - get_accum_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; - get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (use_immed_pkt) { - void *src = (void *) origin_addr, *dest = (void *) (get_accum_pkt->info.data); - mpi_errno = immed_copy(src, dest, orig_data_sz); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - } + get_accum_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; + get_accum_pkt->count = target_count; + get_accum_pkt->datatype = target_datatype; + get_accum_pkt->info.dataloop_size = 0; + get_accum_pkt->op = op; + get_accum_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; + get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *) origin_addr, *dest = (void *) (get_accum_pkt->info.data); + mpi_errno = immed_copy(src, dest, orig_data_sz); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -1254,6 +1190,10 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, } else { MPIDI_RMA_Op_t *op_ptr = NULL; + MPIDI_CH3_Pkt_fop_t *fop_pkt; + MPI_Aint type_size; + int use_immed_pkt = FALSE; + int is_contig; /* Append this operation to the RMA ops queue */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); @@ -1262,96 +1202,49 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - if (op == MPI_NO_OP) { - /* Convert FOP to a Get */ - MPIDI_CH3_Pkt_get_t *get_pkt; - MPI_Aint target_type_size; - int use_immed_resp_pkt = FALSE; - int is_contig; - - /******************** Setting operation struct areas ***********************/ + /******************** Setting operation struct areas ***********************/ - op_ptr->origin_addr = result_addr; - op_ptr->origin_count = 1; - op_ptr->origin_datatype = datatype; - op_ptr->target_rank = target_rank; - op_ptr->piggyback_lock_candidate = 1; + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = 1; + op_ptr->origin_datatype = datatype; + op_ptr->result_addr = result_addr; + op_ptr->result_datatype = datatype; + op_ptr->target_rank = target_rank; + op_ptr->piggyback_lock_candidate = 1; - /************** Setting packet struct areas in operation ****************/ + /************** Setting packet struct areas in operation ****************/ - MPID_Datatype_get_size_macro(datatype, target_type_size); - MPIU_Assert(target_type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); + MPID_Datatype_get_size_macro(datatype, type_size); + MPIU_Assert(type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); - MPID_Datatype_is_contig(datatype, &is_contig); + MPID_Datatype_is_contig(datatype, &is_contig); - if (is_contig) { - /* Judege if we can use IMMED data for response packet */ - if (target_type_size <= MPIDI_RMA_IMMED_BYTES) - use_immed_resp_pkt = TRUE; + if (is_contig) { + /* Judge if we can use IMMED data packet */ + if (type_size <= MPIDI_RMA_IMMED_BYTES) { + use_immed_pkt = TRUE; } - - get_pkt = &(op_ptr->pkt.get); - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + - win_ptr->basic_info_table[target_rank].disp_unit * target_disp; - get_pkt->count = 1; - get_pkt->datatype = datatype; - get_pkt->info.dataloop_size = 0; - get_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; - get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (use_immed_resp_pkt == TRUE) - get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } - else { - MPIDI_CH3_Pkt_fop_t *fop_pkt; - MPI_Aint type_size; - int use_immed_pkt = FALSE; - int is_contig; - - /******************** Setting operation struct areas ***********************/ - - op_ptr->origin_addr = (void *) origin_addr; - op_ptr->origin_count = 1; - op_ptr->origin_datatype = datatype; - op_ptr->result_addr = result_addr; - op_ptr->result_datatype = datatype; - op_ptr->target_rank = target_rank; - op_ptr->piggyback_lock_candidate = 1; - - /************** Setting packet struct areas in operation ****************/ - MPID_Datatype_get_size_macro(datatype, type_size); - MPIU_Assert(type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); - - MPID_Datatype_is_contig(datatype, &is_contig); - - if (is_contig) { - /* Judge if we can use IMMED data packet */ - if (type_size <= MPIDI_RMA_IMMED_BYTES) { - use_immed_pkt = TRUE; - } - } + fop_pkt = &(op_ptr->pkt.fop); - fop_pkt = &(op_ptr->pkt.fop); - - if (use_immed_pkt) { - MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP_IMMED); - } - else { - MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); - } - fop_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + - win_ptr->basic_info_table[target_rank].disp_unit * target_disp; - fop_pkt->datatype = datatype; - fop_pkt->op = op; - fop_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; - fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (use_immed_pkt) { - void *src = (void *) origin_addr, *dest = (void *) (fop_pkt->info.data); - mpi_errno = immed_copy(src, dest, type_size); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - } + if (use_immed_pkt) { + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP_IMMED); + } + else { + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + } + fop_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; + fop_pkt->datatype = datatype; + fop_pkt->op = op; + fop_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; + fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *) origin_addr, *dest = (void *) (fop_pkt->info.data); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) + MPIU_ERR_POP(mpi_errno); } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); http://git.mpich.org/mpich.git/commitdiff/22c16664e41a0f12eb984822909b8c452d... commit 22c16664e41a0f12eb984822909b8c452d066f63 Author: Xin Zhao <[email protected]> Date: Fri May 22 14:50:02 2015 -0500 Bug-fix: datatype in FOP should be predefined instead of builtin Originally MPICH check datatype of FOP by judging if it is a BUILTIN type, this prohibits all pair types. This patch fix this issue. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpi/rma/fetch_and_op.c b/src/mpi/rma/fetch_and_op.c index 92cede9..6001418 100644 --- a/src/mpi/rma/fetch_and_op.c +++ b/src/mpi/rma/fetch_and_op.c @@ -125,7 +125,7 @@ int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIR_ERRTEST_ARGNULL(result_addr, "result_addr", mpi_errno); MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno); - if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) + if (!MPIR_DATATYPE_IS_PREDEFINED(datatype)) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_TYPE, "**typenotpredefined"); } http://git.mpich.org/mpich.git/commitdiff/a59e611783f5fa0a6f3763d910ac0e1fe5... commit a59e611783f5fa0a6f3763d910ac0e1fe57f3350 Author: Xin Zhao <[email protected]> Date: Thu May 28 13:35:20 2015 -0500 Add comments about MPI_NO_OP in GACC/FOP. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpi/rma/fetch_and_op.c b/src/mpi/rma/fetch_and_op.c index 4ce6f8f..92cede9 100644 --- a/src/mpi/rma/fetch_and_op.c +++ b/src/mpi/rma/fetch_and_op.c @@ -117,6 +117,8 @@ int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, if (mpi_errno) goto fn_fail; if (op != MPI_NO_OP) { + /* NOTE: when op is MPI_NO_OP, origin_addr is allowed to be NULL. + * In such case, MPI_Fetch_and_op equals to an atomic GET. */ MPIR_ERRTEST_ARGNULL(origin_addr, "origin_addr", mpi_errno); } diff --git a/src/mpi/rma/get_accumulate.c b/src/mpi/rma/get_accumulate.c index 8f58b20..a9ed4fe 100644 --- a/src/mpi/rma/get_accumulate.c +++ b/src/mpi/rma/get_accumulate.c @@ -137,6 +137,10 @@ int MPI_Get_accumulate(const void *origin_addr, int origin_count, if (mpi_errno) goto fn_fail; if (op != MPI_NO_OP) { + /* NOTE: when op is MPI_NO_OP, origin_addr is allowed to be NULL, + * origin_datatype is allowed to be MPI_DATATYPE_NULL, and + * origin_count is allowed to be 0. In such case, MPI_Get_accumulate + * equals to an atomic GET. */ MPIR_ERRTEST_COUNT(origin_count, mpi_errno); MPIR_ERRTEST_DATATYPE(origin_datatype, "origin_datatype", mpi_errno); MPIR_ERRTEST_USERBUFFER(origin_addr, origin_count, origin_datatype, mpi_errno); http://git.mpich.org/mpich.git/commitdiff/56ede5541807579d6d4c4577a7cf21a3c1... commit 56ede5541807579d6d4c4577a7cf21a3c12f1249 Author: Xin Zhao <[email protected]> Date: Thu May 28 15:58:47 2015 -0500 Delete redundant branch for MPI_REPLACE. Originally, when op is MPI_REPLACE, MPICH does the related work in a separate IF branch different with other OPs in MPIR_Op_table[], which is unnecessary, since MPI_REPLACE has already been implemented in MPIR_Op_table[] (in the same way with the code in that IF branch). Here we delete the redundant IF branch but just trigger MPI_REPLACE from MPIR_Op_table[]. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 42b8c42..db1e7ec 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -824,19 +824,17 @@ static inline int do_accumulate_op(void *source_buf, int source_count, MPI_Datat MPID_Datatype_get_extent_macro(source_dtp, source_dtp_extent); } - if (acc_op != MPI_REPLACE) { - if (HANDLE_GET_KIND(acc_op) == HANDLE_KIND_BUILTIN) { - /* get the function by indexing into the op table */ - uop = MPIR_OP_HDL_TO_FN(acc_op); - } - else { - /* --BEGIN ERROR HANDLING-- */ - mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, - FCNAME, __LINE__, MPI_ERR_OP, - "**opnotpredefined", "**opnotpredefined %d", acc_op); - return mpi_errno; - /* --END ERROR HANDLING-- */ - } + if (HANDLE_GET_KIND(acc_op) == HANDLE_KIND_BUILTIN) { + /* get the function by indexing into the op table */ + uop = MPIR_OP_HDL_TO_FN(acc_op); + } + else { + /* --BEGIN ERROR HANDLING-- */ + mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, + FCNAME, __LINE__, MPI_ERR_OP, + "**opnotpredefined", "**opnotpredefined %d", acc_op); + return mpi_errno; + /* --END ERROR HANDLING-- */ } @@ -854,15 +852,7 @@ static inline int do_accumulate_op(void *source_buf, int source_count, MPI_Datat curr_target_buf = target_buf; } - if (acc_op == MPI_REPLACE) { - mpi_errno = MPIR_Localcopy(source_buf, source_count, source_dtp, - curr_target_buf, source_count, source_dtp); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - } - else { - (*uop) (source_buf, curr_target_buf, &source_count, &source_dtp); - } + (*uop) (source_buf, curr_target_buf, &source_count, &source_dtp); } else { /* derived datatype */ @@ -929,18 +919,8 @@ static inline int do_accumulate_op(void *source_buf, int source_count, MPI_Datat MPIU_Assign_trunc(count, curr_len / type_size, int); - if (acc_op == MPI_REPLACE) { - mpi_errno = MPIR_Localcopy((char *) source_buf + type_extent * accumulated_count, - count, type, - (char *) target_buf + MPIU_PtrToAint(curr_loc), - count, type); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - } - else { - (*uop) ((char *) source_buf + type_extent * accumulated_count, - (char *) target_buf + MPIU_PtrToAint(curr_loc), &count, &type); - } + (*uop) ((char *) source_buf + type_extent * accumulated_count, + (char *) target_buf + MPIU_PtrToAint(curr_loc), &count, &type); if (curr_len % type_size == 0) { i++; http://git.mpich.org/mpich.git/commitdiff/37cb95b14b693b634eb80fca442ff809ac... commit 37cb95b14b693b634eb80fca442ff809acab10ba Author: Xin Zhao <[email protected]> Date: Tue May 19 21:57:08 2015 -0500 Make do_accumulate_op() perform correctly when op is MPI_NO_OP. Originally before do_accumulate_op() is called, there is a separate IF branch to deal with MPI_NO_OP, which is gross. Here we modify do_accumulate_op() to make sure it works correctly for MPI_NO_OP, and delete those IF branches. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_shm.h b/src/mpid/ch3/include/mpid_rma_shm.h index a34e64f..1ad0351 100644 --- a/src/mpid/ch3/include/mpid_rma_shm.h +++ b/src/mpid/ch3/include/mpid_rma_shm.h @@ -439,11 +439,16 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ MPI_Aint total_len, rest_len; MPI_Aint origin_dtp_size; MPID_Datatype *origin_dtp_ptr = NULL; + int is_empty_origin = FALSE; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_GET_ACC_OP); + /* Judge if origin buffer is empty */ + if (op == MPI_NO_OP) + is_empty_origin = TRUE; + if (win_ptr->shm_allocated == TRUE) { int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank]; MPIU_Assert(local_target_rank >= 0); @@ -464,17 +469,7 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_ MPIU_ERR_POP(mpi_errno); } - /* NO_OP: Don't perform the accumulate */ - if (op == MPI_NO_OP) { - if (shm_locked) { - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - shm_locked = 0; - } - - goto fn_exit; - } - - if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { + if (is_empty_origin == TRUE || MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { mpi_errno = do_accumulate_op((void*)origin_addr, origin_count, origin_datatype, (void*)((char *)base+disp_unit*target_disp), target_count, target_datatype, diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 0d80984..42b8c42 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -808,14 +808,21 @@ static inline int do_accumulate_op(void *source_buf, int source_count, MPI_Datat { int mpi_errno = MPI_SUCCESS; MPI_User_function *uop = NULL; - MPI_Aint source_dtp_size, source_dtp_extent; + MPI_Aint source_dtp_size = 0, source_dtp_extent = 0; + int is_empty_source = FALSE; MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP); MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP); - MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(source_dtp)); - MPID_Datatype_get_size_macro(source_dtp, source_dtp_size); - MPID_Datatype_get_extent_macro(source_dtp, source_dtp_extent); + /* first Judge if source buffer is empty */ + if (acc_op == MPI_NO_OP) + is_empty_source = TRUE; + + if (is_empty_source == FALSE) { + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(source_dtp)); + MPID_Datatype_get_size_macro(source_dtp, source_dtp_size); + MPID_Datatype_get_extent_macro(source_dtp, source_dtp_extent); + } if (acc_op != MPI_REPLACE) { if (HANDLE_GET_KIND(acc_op) == HANDLE_KIND_BUILTIN) { @@ -833,13 +840,19 @@ static inline int do_accumulate_op(void *source_buf, int source_count, MPI_Datat } - if (MPIR_DATATYPE_IS_PREDEFINED(target_dtp)) { - /* apply op if target dtp is predefined dtp */ - - MPIU_Assert(source_dtp == target_dtp); + if (is_empty_source == TRUE || MPIR_DATATYPE_IS_PREDEFINED(target_dtp)) { + /* directly apply op if target dtp is predefined dtp OR source buffer is empty */ + MPI_Aint real_stream_offset; + void *curr_target_buf; - MPI_Aint real_stream_offset = (stream_offset / source_dtp_size) * source_dtp_extent; - void *curr_target_buf = (void *) ((char *) target_buf + real_stream_offset); + if (is_empty_source == FALSE) { + MPIU_Assert(source_dtp == target_dtp); + real_stream_offset = (stream_offset / source_dtp_size) * source_dtp_extent; + curr_target_buf = (void *) ((char *) target_buf + real_stream_offset); + } + else { + curr_target_buf = target_buf; + } if (acc_op == MPI_REPLACE) { mpi_errno = MPIR_Localcopy(source_buf, source_count, source_dtp, diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index e7aebdb..b2c6e29 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -445,11 +445,8 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i } /* Perform accumulate computation */ - if (rreq->dev.op != MPI_NO_OP) { - mpi_errno = do_accumulate_op(rreq->dev.user_buf, 1, rreq->dev.datatype, - rreq->dev.real_user_buf, 1, rreq->dev.datatype, 0, - rreq->dev.op); - } + mpi_errno = do_accumulate_op(rreq->dev.user_buf, 1, rreq->dev.datatype, + rreq->dev.real_user_buf, 1, rreq->dev.datatype, 0, rreq->dev.op); if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); @@ -1514,15 +1511,13 @@ static inline int perform_fop_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e } /* Apply the op */ - if (fop_pkt->op != MPI_NO_OP) { - if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { - mpi_errno = do_accumulate_op(fop_pkt->info.data, 1, fop_pkt->datatype, - fop_pkt->addr, 1, fop_pkt->datatype, 0, fop_pkt->op); - } - else { - mpi_errno = do_accumulate_op(lock_entry->data, 1, fop_pkt->datatype, - fop_pkt->addr, 1, fop_pkt->datatype, 0, fop_pkt->op); - } + if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { + mpi_errno = do_accumulate_op(fop_pkt->info.data, 1, fop_pkt->datatype, + fop_pkt->addr, 1, fop_pkt->datatype, 0, fop_pkt->op); + } + else { + mpi_errno = do_accumulate_op(lock_entry->data, 1, fop_pkt->datatype, + fop_pkt->addr, 1, fop_pkt->datatype, 0, fop_pkt->op); } if (win_ptr->shm_allocated == TRUE) diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 54e953b..29b00ba 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -1356,10 +1356,8 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } /* Apply the op */ - if (fop_pkt->op != MPI_NO_OP) { - mpi_errno = do_accumulate_op(fop_pkt->info.data, 1, fop_pkt->datatype, - fop_pkt->addr, 1, fop_pkt->datatype, 0, fop_pkt->op); - } + mpi_errno = do_accumulate_op(fop_pkt->info.data, 1, fop_pkt->datatype, + fop_pkt->addr, 1, fop_pkt->datatype, 0, fop_pkt->op); if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); http://git.mpich.org/mpich.git/commitdiff/092d6c8ec115ca7fcbdd55f3879cbdc2dc... commit 092d6c8ec115ca7fcbdd55f3879cbdc2dc8a969d Author: Xin Zhao <[email protected]> Date: Fri May 15 15:27:36 2015 -0500 Exit immed_copy() if src/dest address is NULL or data length is zero. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 6069ffc..6eb300e 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -27,6 +27,9 @@ static inline int immed_copy(void *src, void *dest, size_t len) MPIDI_FUNC_ENTER(MPID_STATE_IMMED_COPY); + if (src == NULL || dest == NULL || len == 0) + goto fn_exit; + switch (len) { case 1: *(uint8_t *) dest = *(uint8_t *) src; http://git.mpich.org/mpich.git/commitdiff/69c2207df8e81625f0df1685a2242da93e... commit 69c2207df8e81625f0df1685a2242da93e5380cd Author: Xin Zhao <[email protected]> Date: Wed May 27 10:03:09 2015 -0500 Delete redundant function call. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index c32de97..dd91a12 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -501,27 +501,6 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_win(MPID_Win * win_ptr, #undef FUNCNAME -#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_single_target -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Cleanup_single_target(MPID_Win * win_ptr, - MPIDI_RMA_Target_t * target) -{ - int mpi_errno = MPI_SUCCESS; - - /* dequeue the target and free it. */ - mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, target); - if (mpi_errno != MPI_SUCCESS) - MPIU_ERR_POP(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME #define FUNCNAME MPIDI_CH3I_RMA_Cleanup_targets_win #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) @@ -533,7 +512,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_targets_win(MPID_Win * win_ptr) for (i = 0; i < win_ptr->num_slots; i++) { for (target = win_ptr->slots[i].target_list_head; target;) { next_target = target->next; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_single_target(win_ptr, target); + mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); target = next_target; diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 7a6829c..e106861 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -759,7 +759,7 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe } while (!remote_completed); /* Cleanup the target. */ - mpi_errno = MPIDI_CH3I_RMA_Cleanup_single_target(win_ptr, curr_target); + mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, curr_target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 43efcea..731a94c 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -1132,7 +1132,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) } /* Cleanup the target. */ - mpi_errno = MPIDI_CH3I_RMA_Cleanup_single_target(win_ptr, target); + mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } http://git.mpich.org/mpich.git/commitdiff/a7be0681b9ed8e4a49dec90f7de2257441... commit a7be0681b9ed8e4a49dec90f7de22574410027ed Author: Xin Zhao <[email protected]> Date: Fri May 22 11:36:12 2015 -0500 Code-refactoring: move completion judgement code out of GC function. In this patch, we move the judgement of local/remote completion out of GC function to separate macros, so that GC function only does the garbage collection work. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index ebec7f0..c32de97 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -22,6 +22,26 @@ extern MPIDI_RMA_Target_t *global_rma_target_pool, *global_rma_target_pool_tail, MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc); +#define MPIDI_CH3I_RMA_ops_completion(win_, target_, local_completed_, remote_completed_) \ + do { \ + local_completed_ = 0; \ + remote_completed_ = 0; \ + if (win_->states.access_state != MPIDI_RMA_FENCE_ISSUED && \ + win_->states.access_state != MPIDI_RMA_PSCW_ISSUED && \ + win_->states.access_state != MPIDI_RMA_LOCK_ALL_ISSUED && \ + target_->access_state != MPIDI_RMA_LOCK_CALLED && \ + target_->access_state != MPIDI_RMA_LOCK_ISSUED && \ + target_->pending_op_list_head == NULL && \ + target_->issued_read_op_list_head == NULL && \ + target_->issued_write_op_list_head == NULL && \ + target_->issued_dt_op_list_head == NULL) { \ + local_completed_ = 1; \ + if (target_->sync.sync_flag == MPIDI_RMA_SYNC_NONE && \ + target_->sync.outstanding_acks == 0) \ + remote_completed_ = 1; \ + } \ + } while (0) + /* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and * initialize it. If we cannot get one, return NULL. */ #undef FUNCNAME @@ -329,8 +349,7 @@ static inline int MPIDI_CH3I_Win_target_dequeue_and_free(MPID_Win * win_ptr, MPI #define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_target #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, - int *local_completed, int *remote_completed) +static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target) { MPIDI_RMA_Op_t *curr_op = NULL; MPIDI_RMA_Op_t **op_list_head = NULL, **op_list_tail = NULL; @@ -338,9 +357,6 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM int mpi_errno = MPI_SUCCESS; int i; - (*local_completed) = 0; - (*remote_completed) = 0; - if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED || win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED || win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) @@ -356,7 +372,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM if (target->pending_op_list_head == NULL && target->issued_read_op_list_head == NULL && target->issued_write_op_list_head == NULL && target->issued_dt_op_list_head == NULL) - goto cleanup_target; + goto fn_exit; /* go over issued_read_op_list, issued_write_op_list, * issued_dt_op_list, start from issued_read_op_list. */ @@ -433,21 +449,6 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM curr_op = *op_list_head; } - cleanup_target: - if (target->pending_op_list_head == NULL && - target->issued_read_op_list_head == NULL && target->issued_write_op_list_head == NULL && - target->issued_dt_op_list_head == NULL) { - - (*local_completed) = 1; - - /* for the conditions that need to be satisfied before we free the - * target, see the MPIDI_RMA_Target definition in - * mpid_rma_types.h */ - if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && target->sync.outstanding_acks == 0) { - (*remote_completed) = 1; - } - } - fn_exit: return mpi_errno; fn_fail: @@ -473,10 +474,12 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_win(MPID_Win * win_ptr, for (target = win_ptr->slots[i].target_list_head; target;) { int local = 0, remote = 0; - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target, &local, &remote); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local, remote); + num_targets++; local_completed_targets += local; remote_completed_targets += remote; diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 3a6e6e9..7a6829c 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -628,7 +628,7 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr) { - int i, local_completed = 0, remote_completed = 0; + int i, local_completed = 0, remote_completed ATTRIBUTE((unused)) = 0; int mpi_errno = MPI_SUCCESS; MPIDI_RMA_Target_t *curr_target = NULL; int made_progress = 0; @@ -663,10 +663,12 @@ int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr) /* Wait for local completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target, - &local_completed, &remote_completed); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed); + if (!local_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) @@ -687,7 +689,7 @@ int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr) #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Target_t ** target) { - int i, local_completed = 0, remote_completed = 0; + int i, local_completed ATTRIBUTE((unused)) = 0, remote_completed = 0; int made_progress = 0; MPIDI_RMA_Target_t *curr_target = NULL; int mpi_errno = MPI_SUCCESS; @@ -743,10 +745,12 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target, - &local_completed, &remote_completed); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed); + if (!remote_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 76b8fd5..43efcea 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -1050,7 +1050,7 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win * win_ptr) int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) { int made_progress = 0; - int local_completed = 0, remote_completed = 0; + int local_completed ATTRIBUTE((unused)) = 0, remote_completed = 0; MPIDI_RMA_Target_t *target = NULL; enum MPIDI_RMA_sync_types sync_flag; int progress_engine_triggered = 0; @@ -1098,10 +1098,12 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target, - &local_completed, &remote_completed); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local_completed, remote_completed); + if (!remote_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) @@ -1165,7 +1167,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) int MPIDI_Win_flush(int dest, MPID_Win * win_ptr) { int made_progress = 0; - int local_completed = 0, remote_completed = 0; + int local_completed ATTRIBUTE((unused)) = 0, remote_completed = 0; int rank = win_ptr->comm_ptr->rank; MPIDI_RMA_Target_t *target = NULL; int progress_engine_triggered = 0; @@ -1218,10 +1220,12 @@ int MPIDI_Win_flush(int dest, MPID_Win * win_ptr) /* Wait for remote completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target, - &local_completed, &remote_completed); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local_completed, remote_completed); + if (!remote_completed) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) @@ -1329,10 +1333,12 @@ int MPIDI_Win_flush_local(int dest, MPID_Win * win_ptr) /* Wait for local completion. */ do { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target, - &local_completed, &remote_completed); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + MPIDI_CH3I_RMA_ops_completion(win_ptr, target, local_completed, remote_completed); + if ((target->sync.upgrade_flush_local && !remote_completed) || (!target->sync.upgrade_flush_local && !local_completed)) { mpi_errno = wait_progress_engine(); @@ -1790,11 +1796,13 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) for (i = 0; i < win_ptr->num_slots; i++) { curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { - mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target, - &local_completed, &remote_completed); + mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, + remote_completed); + if (curr_target->sync.upgrade_flush_local) { if (remote_completed) { remote_completed_cnt++; http://git.mpich.org/mpich.git/commitdiff/77c07868c0414781bae758ba8a7693287a... commit 77c07868c0414781bae758ba8a7693287a1a24eb Author: Xin Zhao <[email protected]> Date: Wed May 20 15:45:23 2015 -0500 Add namespace for num_active_issued_win and num_passive_win. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h b/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h index 22baa82..15188ca 100644 --- a/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h +++ b/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h @@ -40,7 +40,8 @@ static inline void MPID_nem_mpich_send_seg (MPID_Segment *segment, MPIDI_msg_sz_ !MPIDI_CH3I_shm_active_send && \ !MPIDI_CH3I_Sendq_head(MPIDI_CH3I_shm_sendq) && \ !MPIDU_Sched_are_pending() && \ - !num_active_issued_win && !num_passive_win) + !MPIDI_CH3I_num_active_issued_win && \ + !MPIDI_CH3I_num_passive_win) #undef FUNCNAME #define FUNCNAME MPID_nem_mpich_send_header diff --git a/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_pre.h b/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_pre.h index 8ab69cb..6a01926 100644 --- a/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_pre.h +++ b/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_pre.h @@ -228,7 +228,7 @@ MPIDI_CH3I_Progress_state; #define MPIDI_CH3_PROGRESS_STATE_DECL MPIDI_CH3I_Progress_state ch; extern OPA_int_t MPIDI_CH3I_progress_completion_count; -extern int num_active_issued_win, num_passive_win; +extern int MPIDI_CH3I_num_active_issued_win, MPIDI_CH3I_num_passive_win; #define MPIDI_CH3I_INCR_PROGRESS_COMPLETION_COUNT do { \ OPA_write_barrier(); \ diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c index 6e699b7..e05fe3b 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c @@ -46,7 +46,7 @@ extern MPID_Request ** const MPID_Recvq_unexpected_tail_ptr; #endif OPA_int_t MPIDI_CH3I_progress_completion_count = OPA_INT_T_INITIALIZER(0); -int num_active_issued_win = 0, num_passive_win = 0; +int MPIDI_CH3I_num_active_issued_win = 0, MPIDI_CH3I_num_passive_win = 0; /* NEMESIS MULTITHREADING: Extra Data Structures Added */ #ifdef MPICH_IS_THREADED @@ -478,7 +478,7 @@ int MPIDI_CH3I_Progress (MPID_Progress_state *progress_state, int is_blocking) #endif /* HAVE_LIBHCOLL */ /* make progress on RMA */ - if (num_active_issued_win > 0 || num_passive_win > 0) { + if (MPIDI_CH3I_num_active_issued_win > 0 || MPIDI_CH3I_num_passive_win > 0) { mpi_errno = MPIDI_CH3I_RMA_Make_progress_global(&made_progress); if (mpi_errno) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/channels/sock/include/mpidi_ch3_pre.h b/src/mpid/ch3/channels/sock/include/mpidi_ch3_pre.h index 6e965d0..fc1115c 100644 --- a/src/mpid/ch3/channels/sock/include/mpidi_ch3_pre.h +++ b/src/mpid/ch3/channels/sock/include/mpidi_ch3_pre.h @@ -78,7 +78,7 @@ MPIDI_CH3I_Progress_state; and must be available to the routines in src/mpi */ extern volatile unsigned int MPIDI_CH3I_progress_completion_count; -extern int num_active_issued_win, num_passive_win; +extern int MPIDI_CH3I_num_active_issued_win, MPIDI_CH3I_num_passive_win; /* MPICH_IS_THREADED isn't defined yet (handled by mpiimplthread.h) */ #if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE) diff --git a/src/mpid/ch3/channels/sock/src/ch3_progress.c b/src/mpid/ch3/channels/sock/src/ch3_progress.c index 6c649db..8715193 100644 --- a/src/mpid/ch3/channels/sock/src/ch3_progress.c +++ b/src/mpid/ch3/channels/sock/src/ch3_progress.c @@ -22,7 +22,7 @@ static int MPIDI_CH3i_Progress_test(void); /* FIXME: Move thread stuff into some set of abstractions in order to remove ifdefs */ volatile unsigned int MPIDI_CH3I_progress_completion_count = 0; -int num_active_issued_win = 0, num_passive_win = 0; +int MPIDI_CH3I_num_active_issued_win = 0, MPIDI_CH3I_num_passive_win = 0; #ifdef MPICH_IS_THREADED volatile int MPIDI_CH3I_progress_blocked = FALSE; @@ -98,7 +98,7 @@ static int MPIDI_CH3i_Progress_test(void) #endif /* HAVE_LIBHCOLL */ /* make progress on RMA */ - if (num_active_issued_win > 0 || num_passive_win > 0) { + if (MPIDI_CH3I_num_active_issued_win > 0 || MPIDI_CH3I_num_passive_win > 0) { mpi_errno = MPIDI_CH3I_RMA_Make_progress_global(&made_progress); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -213,7 +213,7 @@ static int MPIDI_CH3i_Progress_wait(MPID_Progress_state * progress_state) #endif /* HAVE_LIBHCOLL */ /* make progress on RMA */ - if (num_active_issued_win > 0 || num_passive_win > 0) { + if (MPIDI_CH3I_num_active_issued_win > 0 || MPIDI_CH3I_num_passive_win > 0) { mpi_errno = MPIDI_CH3I_RMA_Make_progress_global(&made_progress); if (mpi_errno) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index dd12b96..3a6e6e9 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -62,8 +62,8 @@ static inline int check_window_state(MPID_Win * win_ptr, int *made_progress) MPID_Request_release(fence_req_ptr); win_ptr->fence_sync_req = MPI_REQUEST_NULL; - num_active_issued_win--; - MPIU_Assert(num_active_issued_win >= 0); + MPIDI_CH3I_num_active_issued_win--; + MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0); (*made_progress) = 1; } @@ -75,8 +75,8 @@ static inline int check_window_state(MPID_Win * win_ptr, int *made_progress) * we do not create PSCW requests on window. */ win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED; - num_active_issued_win--; - MPIU_Assert(num_active_issued_win >= 0); + MPIDI_CH3I_num_active_issued_win--; + MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0); (*made_progress) = 1; } @@ -98,8 +98,8 @@ static inline int check_window_state(MPID_Win * win_ptr, int *made_progress) if (i == win_ptr->start_grp_size) { win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED; - num_active_issued_win--; - MPIU_Assert(num_active_issued_win >= 0); + MPIDI_CH3I_num_active_issued_win--; + MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0); (*made_progress) = 1; diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index c73c356..76b8fd5 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -348,7 +348,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) /* Set window access state properly. */ win_ptr->states.access_state = MPIDI_RMA_FENCE_ISSUED; - num_active_issued_win++; + MPIDI_CH3I_num_active_issued_win++; goto finish_fence; } @@ -671,7 +671,7 @@ int MPIDI_Win_start(MPID_Group * group_ptr, int assert, MPID_Win * win_ptr) finish_start: /* Set window access state properly. */ win_ptr->states.access_state = MPIDI_RMA_PSCW_ISSUED; - num_active_issued_win++; + MPIDI_CH3I_num_active_issued_win++; /* BEGINNING synchronization: the following counter should be zero. */ MPIU_Assert(win_ptr->accumulated_ops_cnt == 0); @@ -981,7 +981,7 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win * win_ptr) if (win_ptr->lock_epoch_count == 0) { /* Set window access state properly. */ win_ptr->states.access_state = MPIDI_RMA_PER_TARGET; - num_passive_win++; + MPIDI_CH3I_num_passive_win++; } win_ptr->lock_epoch_count++; @@ -1118,8 +1118,8 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) if (win_ptr->lock_epoch_count == 0) { /* Set window access state properly. */ win_ptr->states.access_state = MPIDI_RMA_NONE; - num_passive_win--; - MPIU_Assert(num_passive_win >= 0); + MPIDI_CH3I_num_passive_win--; + MPIU_Assert(MPIDI_CH3I_num_passive_win >= 0); } if (target != NULL) { @@ -1406,7 +1406,7 @@ int MPIDI_Win_lock_all(int assert, MPID_Win * win_ptr) win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_GRANTED; else win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_CALLED; - num_passive_win++; + MPIDI_CH3I_num_passive_win++; win_ptr->lock_all_assert = assert; @@ -1597,8 +1597,8 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr) finish_unlock_all: /* Set window access state properly. */ win_ptr->states.access_state = MPIDI_RMA_NONE; - num_passive_win--; - MPIU_Assert(num_passive_win >= 0); + MPIDI_CH3I_num_passive_win--; + MPIU_Assert(MPIDI_CH3I_num_passive_win >= 0); /* reset lock_all assert on window. */ win_ptr->lock_all_assert = 0; http://git.mpich.org/mpich.git/commitdiff/9fb9c2d36a51c55208036c3c4105dde8a0... commit 9fb9c2d36a51c55208036c3c4105dde8a004c84e Author: Xin Zhao <[email protected]> Date: Wed May 20 14:23:42 2015 -0500 Fix typo error on variable names. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 2bdc5a5..3d5568e 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -805,10 +805,10 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_get_ptr(origin_datatype, origin_dtp); } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { - MPID_Datatype_get_ptr(result_datatype, target_dtp); + MPID_Datatype_get_ptr(result_datatype, result_dtp); } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { - MPID_Datatype_get_ptr(target_datatype, result_dtp); + MPID_Datatype_get_ptr(target_datatype, target_dtp); } MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); http://git.mpich.org/mpich.git/commitdiff/0185437d948d81167a7aaf1e9dd02c4de7... commit 0185437d948d81167a7aaf1e9dd02c4de7b891c8 Author: Xin Zhao <[email protected]> Date: Tue May 19 21:08:56 2015 -0500 Change variable name of origin data_sz and target data_sz. Originally in GACC/GET, we use 'data_sz' for origin data size and 'len/orig_len' for target data size. Here we change them to 'orig_data_sz' for origin data size and 'target_data_sz' for target data size to make it clearer. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index ddcc4a6..2bdc5a5 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -247,7 +247,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPID_Request * ureq) { int mpi_errno = MPI_SUCCESS; - MPIDI_msg_sz_t data_sz; + MPIDI_msg_sz_t orig_data_sz, target_data_sz; int dt_contig ATTRIBUTE((unused)), rank; MPI_Aint dt_true_lb ATTRIBUTE((unused)); MPID_Datatype *dtp; @@ -264,9 +264,10 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype goto fn_exit; } - MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb); + MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, orig_data_sz, dtp, + dt_true_lb); - if (data_sz == 0) { + if (orig_data_sz == 0) { goto fn_exit; } @@ -304,7 +305,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; MPI_Aint target_type_size; - size_t len; int use_immed_resp_pkt = FALSE; int is_origin_contig, is_target_contig; @@ -341,12 +341,12 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPID_Datatype_is_contig(target_datatype, &is_target_contig); MPID_Datatype_get_size_macro(target_datatype, target_type_size); - MPIU_Assign_trunc(len, target_count * target_type_size, size_t); + MPIU_Assign_trunc(target_data_sz, target_count * target_type_size, MPIDI_msg_sz_t); /* Judge if we can use IMMED data response packet */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { - if (len <= MPIDI_RMA_IMMED_BYTES) + if (target_data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } @@ -647,7 +647,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Win * win_ptr, MPID_Request * ureq) { int mpi_errno = MPI_SUCCESS; - MPIDI_msg_sz_t data_sz; + MPIDI_msg_sz_t orig_data_sz, target_data_sz; int rank; int dt_contig ATTRIBUTE((unused)); MPI_Aint dt_true_lb ATTRIBUTE((unused)); @@ -665,9 +665,10 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, goto fn_exit; } - MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb); + MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, target_data_sz, dtp, + dt_true_lb); - if (data_sz == 0) { + if (target_data_sz == 0) { goto fn_exit; } @@ -747,7 +748,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_result_contig && is_target_contig) { - if (data_sz <= MPIDI_RMA_IMMED_BYTES) + if (target_data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } @@ -778,7 +779,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, else { MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; MPI_Aint origin_type_size; - size_t orig_len; int use_immed_pkt = FALSE; int is_origin_contig, is_target_contig, is_result_contig; MPI_Aint stream_elem_count, stream_unit_count; @@ -812,7 +812,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, } MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); - MPIU_Assign_trunc(orig_len, origin_count * origin_type_size, size_t); + MPIU_Assign_trunc(orig_data_sz, origin_count * origin_type_size, MPIDI_msg_sz_t); /* Get size and count for predefined datatype elements */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -823,7 +823,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, else { MPIU_Assert(origin_dtp->basic_type != MPI_DATATYPE_NULL); MPID_Datatype_get_size_macro(origin_dtp->basic_type, predefined_dtp_size); - predefined_dtp_count = orig_len / predefined_dtp_size; + predefined_dtp_count = orig_data_sz / predefined_dtp_size; MPID_Datatype_get_extent_macro(origin_dtp->basic_type, predefined_dtp_extent); } MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 && @@ -856,7 +856,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig && is_result_contig) { - if (orig_len <= MPIDI_RMA_IMMED_BYTES) + if (orig_data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } @@ -867,7 +867,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for origin, target and result data. We should extend this optimization to derived * datatypes as well. */ - if (orig_len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) + if (orig_data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) op_ptr->piggyback_lock_candidate = 1; } @@ -892,7 +892,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { void *src = (void *) origin_addr, *dest = (void *) (get_accum_pkt->info.data); - mpi_errno = immed_copy(src, dest, orig_len); + mpi_errno = immed_copy(src, dest, orig_data_sz); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } http://git.mpich.org/mpich.git/commitdiff/5b1b3bbf6ef9b7c8d54e9fea202357e23d... commit 5b1b3bbf6ef9b7c8d54e9fea202357e23d7cf243 Author: Xin Zhao <[email protected]> Date: Tue May 19 21:06:37 2015 -0500 Delete redundant variable "len". Since both "data_sz" and "len" store the same data (the length of the entire origin data), here we delete "len" but just use "data_sz". Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index a8fd127..ddcc4a6 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -117,8 +117,6 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_put_t *put_pkt = NULL; - MPI_Aint origin_type_size; - size_t len; int use_immed_pkt = FALSE; int is_origin_contig, is_target_contig; @@ -154,13 +152,10 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); MPID_Datatype_is_contig(target_datatype, &is_target_contig); - MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); - /* Judge if we can use IMMED data packet */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { - if (len <= MPIDI_RMA_IMMED_BYTES) + if (data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } @@ -170,7 +165,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ - if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) + if (data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) op_ptr->piggyback_lock_candidate = 1; } @@ -195,7 +190,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { void *src = (void *) origin_addr, *dest = (void *) (put_pkt->info.data); - mpi_errno = immed_copy(src, dest, len); + mpi_errno = immed_copy(src, dest, data_sz); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -488,8 +483,6 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp else { MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; - MPI_Aint origin_type_size; - size_t len; int use_immed_pkt = FALSE; int is_origin_contig, is_target_contig; MPI_Aint stream_elem_count, stream_unit_count; @@ -523,19 +516,16 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPID_Datatype_get_ptr(target_datatype, target_dtp); } - MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); - /* Get size and count for predefined datatype elements */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { - predefined_dtp_size = origin_type_size; + MPID_Datatype_get_size_macro(origin_datatype, predefined_dtp_size); predefined_dtp_count = origin_count; MPID_Datatype_get_extent_macro(origin_datatype, predefined_dtp_extent); } else { MPIU_Assert(origin_dtp->basic_type != MPI_DATATYPE_NULL); MPID_Datatype_get_size_macro(origin_dtp->basic_type, predefined_dtp_size); - predefined_dtp_count = len / predefined_dtp_size; + predefined_dtp_count = data_sz / predefined_dtp_size; MPID_Datatype_get_extent_macro(origin_dtp->basic_type, predefined_dtp_extent); } MPIU_Assert(predefined_dtp_count > 0 && predefined_dtp_size > 0 && @@ -562,7 +552,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp /* Judge if we can use IMMED data packet */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { - if (len <= MPIDI_RMA_IMMED_BYTES) + if (data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } @@ -572,7 +562,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ - if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) + if (data_sz <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) op_ptr->piggyback_lock_candidate = 1; } @@ -598,7 +588,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { void *src = (void *) origin_addr, *dest = (void *) (accum_pkt->info.data); - mpi_errno = immed_copy(src, dest, len); + mpi_errno = immed_copy(src, dest, data_sz); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -728,8 +718,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (op == MPI_NO_OP) { /* Convert GAcc to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; - MPI_Aint target_type_size; - size_t len; int use_immed_resp_pkt = FALSE; int is_result_contig, is_target_contig; @@ -752,9 +740,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_add_ref(dtp); } - MPID_Datatype_get_size_macro(target_datatype, target_type_size); - MPIU_Assign_trunc(len, target_count * target_type_size, size_t); - MPID_Datatype_is_contig(result_datatype, &is_result_contig); MPID_Datatype_is_contig(target_datatype, &is_target_contig); @@ -762,7 +747,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_result_contig && is_target_contig) { - if (len <= MPIDI_RMA_IMMED_BYTES) + if (data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } http://git.mpich.org/mpich.git/commitdiff/408c4abf80cad55721d94917971fd85a86... commit 408c4abf80cad55721d94917971fd85a860dbc5a Author: Xin Zhao <[email protected]> Date: Thu May 14 17:16:14 2015 -0500 Simplify code of garbage collection functions. The original code of going over all issued operation lists and cleaning up operations is gross, this patches simplifies the related code. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index bc7315e..ebec7f0 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -358,96 +358,79 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM target->issued_dt_op_list_head == NULL) goto cleanup_target; - if (target->issued_read_op_list_head != NULL) { - op_list_head = &(target->issued_read_op_list_head); - op_list_tail = &(target->issued_read_op_list_tail); - read_flag = 1; - } - else if (target->issued_write_op_list_head != NULL) { - op_list_head = &(target->issued_write_op_list_head); - op_list_tail = &(target->issued_write_op_list_tail); - write_flag = 1; - } - else if (target->issued_dt_op_list_head != NULL) { - op_list_head = &(target->issued_dt_op_list_head); - op_list_tail = &(target->issued_dt_op_list_tail); - } - else { - /* only pending op list is not NULL, nothing we can do here. */ - goto fn_exit; - } + /* go over issued_read_op_list, issued_write_op_list, + * issued_dt_op_list, start from issued_read_op_list. */ + op_list_head = &(target->issued_read_op_list_head); + op_list_tail = &(target->issued_read_op_list_tail); + read_flag = 1; curr_op = *op_list_head; - while (curr_op != NULL) { - for (i = 0; i < curr_op->reqs_size; i++) { - if (curr_op->reqs[i] == NULL) - continue; + while (1) { + if (curr_op != NULL) { + for (i = 0; i < curr_op->reqs_size; i++) { + if (curr_op->reqs[i] == NULL) + continue; + + if (MPID_Request_is_complete(curr_op->reqs[i])) { + /* If there's an error, return it */ + mpi_errno = curr_op->reqs[i]->status.MPI_ERROR; + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); + + /* No errors, free the request */ + MPID_Request_release(curr_op->reqs[i]); + + curr_op->reqs[i] = NULL; + + win_ptr->active_req_cnt--; + } + else + break; + } - if (MPID_Request_is_complete(curr_op->reqs[i])) { - /* If there's an error, return it */ - mpi_errno = curr_op->reqs[i]->status.MPI_ERROR; - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg"); + if (i == curr_op->reqs_size) { + /* Release user request */ + if (curr_op->ureq) { + /* User request must be completed by progress engine */ + MPIU_Assert(MPID_Request_is_complete(curr_op->ureq)); - /* No errors, free the request */ - MPID_Request_release(curr_op->reqs[i]); + /* Release the ch3 ref */ + MPID_Request_release(curr_op->ureq); + } - curr_op->reqs[i] = NULL; + /* free request array in op struct */ + MPIU_Free(curr_op->reqs); + curr_op->reqs = NULL; + curr_op->reqs_size = 0; - win_ptr->active_req_cnt--; + /* dequeue the operation and free it */ + MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op); + MPIDI_CH3I_Win_op_free(win_ptr, curr_op); } else break; } - - if (i == curr_op->reqs_size) { - /* Release user request */ - if (curr_op->ureq) { - /* User request must be completed by progress engine */ - MPIU_Assert(MPID_Request_is_complete(curr_op->ureq)); - - /* Release the ch3 ref */ - MPID_Request_release(curr_op->ureq); + else { + /* current op ptr reaches NULL, move on to the next list */ + if (read_flag == 1) { + read_flag = 0; + op_list_head = &(target->issued_write_op_list_head); + op_list_tail = &(target->issued_write_op_list_tail); + write_flag = 1; } - - /* free request array in op struct */ - MPIU_Free(curr_op->reqs); - curr_op->reqs = NULL; - curr_op->reqs_size = 0; - - /* dequeue the operation and free it */ - MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op); - MPIDI_CH3I_Win_op_free(win_ptr, curr_op); - - if (*op_list_head == NULL) { - if (read_flag == 1) { - read_flag = 0; - if (target->issued_write_op_list_head != NULL) { - op_list_head = &(target->issued_write_op_list_head); - op_list_tail = &(target->issued_write_op_list_tail); - write_flag = 1; - } - else if (target->issued_dt_op_list_head != NULL) { - op_list_head = &(target->issued_dt_op_list_head); - op_list_tail = &(target->issued_dt_op_list_tail); - } - else - break; - } - else if (write_flag == 1) { - write_flag = 0; - if (target->issued_dt_op_list_head != NULL) { - op_list_head = &(target->issued_dt_op_list_head); - op_list_tail = &(target->issued_dt_op_list_tail); - } - else - break; - } + else if (write_flag == 1) { + write_flag = 0; + op_list_head = &(target->issued_dt_op_list_head); + op_list_tail = &(target->issued_dt_op_list_tail); + } + else { + /* we reach the tail of the last operation list (dt_op_list), + * break out. */ + break; } - /* next op */ - curr_op = *op_list_head; } - else - break; + + /* next op */ + curr_op = *op_list_head; } cleanup_target: diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 0a59107..dd12b96 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -574,42 +574,43 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) * must do a Win_flush instead. */ curr_target->sync.upgrade_flush_local = 1; - if (curr_target->issued_read_op_list_head != NULL) { - op_list_head = &curr_target->issued_read_op_list_head; - op_list_tail = &curr_target->issued_read_op_list_tail; - read_flag = 1; - } - else { - op_list_head = &curr_target->issued_write_op_list_head; - op_list_tail = &curr_target->issued_write_op_list_tail; - } + op_list_head = &curr_target->issued_read_op_list_head; + op_list_tail = &curr_target->issued_read_op_list_tail; + read_flag = 1; + + curr_op = *op_list_head; /* free all ops in the list since we do not need to maintain them anymore */ - for (curr_op = *op_list_head; curr_op != NULL;) { - if (curr_op->reqs_size > 0) { - MPIU_Assert(curr_op->reqs != NULL); - for (i = 0; i < curr_op->reqs_size; i++) { - if (curr_op->reqs[i] != NULL) { - MPID_Request_release(curr_op->reqs[i]); - curr_op->reqs[i] = NULL; - win_ptr->active_req_cnt--; + while (1) { + if (curr_op != NULL) { + if (curr_op->reqs_size > 0) { + MPIU_Assert(curr_op->reqs != NULL); + for (i = 0; i < curr_op->reqs_size; i++) { + if (curr_op->reqs[i] != NULL) { + MPID_Request_release(curr_op->reqs[i]); + curr_op->reqs[i] = NULL; + win_ptr->active_req_cnt--; + } } - } - /* free req array in this op */ - MPIU_Free(curr_op->reqs); - curr_op->reqs = NULL; - curr_op->reqs_size = 0; + /* free req array in this op */ + MPIU_Free(curr_op->reqs); + curr_op->reqs = NULL; + curr_op->reqs_size = 0; + } + MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op); + MPIDI_CH3I_Win_op_free(win_ptr, curr_op); } - MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op); - MPIDI_CH3I_Win_op_free(win_ptr, curr_op); - - if (*op_list_head == NULL) { + else { if (read_flag == 1) { op_list_head = &curr_target->issued_write_op_list_head; op_list_head = &curr_target->issued_write_op_list_tail; read_flag = 0; } + else { + /* we reach the tail of write_op_list, break out. */ + break; + } } curr_op = *op_list_head; } http://git.mpich.org/mpich.git/commitdiff/74c2007fd55936f48a7d77d41a27b301ef... commit 74c2007fd55936f48a7d77d41a27b301ef263653 Author: Xin Zhao <[email protected]> Date: Thu May 14 16:32:33 2015 -0500 Move disable_flush_local into sync struct and rename it. Rename "disable_flush_local" to "upgrade_flush_local", which indicates that we upgrade FLUSH_LOCAL to FLUSH. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index f5198cc..bc7315e 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -122,12 +122,12 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr e->lock_type = MPID_LOCK_NONE; e->lock_mode = 0; e->accumulated_ops_cnt = 0; - e->disable_flush_local = 0; e->win_complete_flag = 0; e->put_acc_issued = 0; e->sync.sync_flag = MPIDI_RMA_SYNC_NONE; e->sync.outstanding_acks = 0; + e->sync.upgrade_flush_local = 0; return e; } diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 0bfb49c..120b9dd 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -90,7 +90,6 @@ typedef struct MPIDI_RMA_Target { int lock_type; /* NONE, SHARED, EXCLUSIVE */ int lock_mode; /* e.g., MODE_NO_CHECK */ int accumulated_ops_cnt; - int disable_flush_local; int win_complete_flag; int put_acc_issued; /* indicate if PUT/ACC is issued in this epoch * after the previous synchronization calls. */ @@ -108,6 +107,9 @@ typedef struct MPIDI_RMA_Target { /* packets sent out that we are expecting an ack for */ int outstanding_acks; + + /* Marked when FLUSH_LOCAL is upgraded to FLUSH */ + int upgrade_flush_local; } sync; MPIDI_RMA_Pool_type_t pool_type; diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 1d1f395..0a59107 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -572,7 +572,7 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) /* After we do this, all following Win_flush_local * must do a Win_flush instead. */ - curr_target->disable_flush_local = 1; + curr_target->sync.upgrade_flush_local = 1; if (curr_target->issued_read_op_list_head != NULL) { op_list_head = &curr_target->issued_read_op_list_head; diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 0e06bc8..c73c356 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -1311,7 +1311,7 @@ int MPIDI_Win_flush_local(int dest, MPID_Win * win_ptr) } /* Set sync_flag in sync struct. */ - if (target->disable_flush_local) { + if (target->sync.upgrade_flush_local) { if (target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; target->sync.outstanding_acks++; @@ -1333,8 +1333,8 @@ int MPIDI_Win_flush_local(int dest, MPID_Win * win_ptr) &local_completed, &remote_completed); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if ((target->disable_flush_local && !remote_completed) || - (!target->disable_flush_local && !local_completed)) { + if ((target->sync.upgrade_flush_local && !remote_completed) || + (!target->sync.upgrade_flush_local && !local_completed)) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1343,13 +1343,13 @@ int MPIDI_Win_flush_local(int dest, MPID_Win * win_ptr) * in this function call. */ progress_engine_triggered = 1; } - } while ((target->disable_flush_local && !remote_completed) || - (!target->disable_flush_local && !local_completed)); + } while ((target->sync.upgrade_flush_local && !remote_completed) || + (!target->sync.upgrade_flush_local && !local_completed)); finish_flush_local: if (target != NULL) { - /* reset disable_flush_local flag in target to 0 */ - target->disable_flush_local = 0; + /* reset upgrade_flush_local flag in target to 0 */ + target->sync.upgrade_flush_local = 0; /* ENDING synchronization: correctly decrement the following counters. */ win_ptr->accumulated_ops_cnt -= target->accumulated_ops_cnt; @@ -1735,7 +1735,7 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) int i, made_progress = 0; int local_completed = 0, remote_completed = 0; MPIDI_RMA_Target_t *curr_target = NULL; - int enable_flush_local_cnt = 0, disable_flush_local_cnt = 0; + int enable_flush_local_cnt = 0, upgrade_flush_local_cnt = 0; int remote_completed_cnt = 0, local_completed_cnt = 0; int progress_engine_triggered = 0; int mpi_errno = MPI_SUCCESS; @@ -1758,12 +1758,12 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) for (i = 0; i < win_ptr->num_slots; i++) { curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { - if (curr_target->disable_flush_local) { + if (curr_target->sync.upgrade_flush_local) { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; curr_target->sync.outstanding_acks++; } - disable_flush_local_cnt++; + upgrade_flush_local_cnt++; } else { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL) { @@ -1795,7 +1795,7 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (curr_target->disable_flush_local) { + if (curr_target->sync.upgrade_flush_local) { if (remote_completed) { remote_completed_cnt++; } @@ -1809,7 +1809,7 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) } } - if (remote_completed_cnt < disable_flush_local_cnt || + if (remote_completed_cnt < upgrade_flush_local_cnt || local_completed_cnt < enable_flush_local_cnt) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) @@ -1819,15 +1819,15 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) * in this function call. */ progress_engine_triggered = 1; } - } while (remote_completed_cnt < disable_flush_local_cnt || + } while (remote_completed_cnt < upgrade_flush_local_cnt || local_completed_cnt < enable_flush_local_cnt); finish_flush_local_all: - /* reset disable_flush_local flag in target to 0 */ + /* reset upgrade_flush_local flag in target to 0 */ for (i = 0; i < win_ptr->num_slots; i++) { curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { - curr_target->disable_flush_local = 0; + curr_target->sync.upgrade_flush_local = 0; curr_target = curr_target->next; } } http://git.mpich.org/mpich.git/commitdiff/8a172d8b4cfb7489d54de9bb2fcafcbed9... commit 8a172d8b4cfb7489d54de9bb2fcafcbed98ff592 Author: Xin Zhao <[email protected]> Date: Thu May 14 16:23:16 2015 -0500 Add "issued_" prefix to read_op_list, write_op_list and dt_op_list. "read_op_list", "write_op_list" and "dt_op_list" are used to store RMA operations that are issued but not completed. Here we add prefix to make it clearer. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index e5b3cff..f5198cc 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -111,9 +111,9 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr MPL_LL_DELETE(win_ptr->target_pool, win_ptr->target_pool_tail, e); } - e->read_op_list_head = e->read_op_list_tail = NULL; - e->write_op_list_head = e->write_op_list_tail = NULL; - e->dt_op_list_head = e->dt_op_list_tail = NULL; + e->issued_read_op_list_head = e->issued_read_op_list_tail = NULL; + e->issued_write_op_list_head = e->issued_write_op_list_tail = NULL; + e->issued_dt_op_list_head = e->issued_dt_op_list_tail = NULL; e->pending_op_list_head = e->pending_op_list_tail = NULL; e->next_op_to_issue = NULL; @@ -145,9 +145,9 @@ static inline int MPIDI_CH3I_Win_target_free(MPID_Win * win_ptr, MPIDI_RMA_Targe /* We enqueue elements to the right pool, so when they get freed * at window free time, they won't conflict with the global pool * or other windows */ - MPIU_Assert(e->read_op_list_head == NULL); - MPIU_Assert(e->write_op_list_head == NULL); - MPIU_Assert(e->dt_op_list_head == NULL); + MPIU_Assert(e->issued_read_op_list_head == NULL); + MPIU_Assert(e->issued_write_op_list_head == NULL); + MPIU_Assert(e->issued_dt_op_list_head == NULL); MPIU_Assert(e->pending_op_list_head == NULL); /* use PREPEND when return objects back to the pool @@ -354,23 +354,23 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM goto fn_exit; if (target->pending_op_list_head == NULL && - target->read_op_list_head == NULL && target->write_op_list_head == NULL && - target->dt_op_list_head == NULL) + target->issued_read_op_list_head == NULL && target->issued_write_op_list_head == NULL && + target->issued_dt_op_list_head == NULL) goto cleanup_target; - if (target->read_op_list_head != NULL) { - op_list_head = &(target->read_op_list_head); - op_list_tail = &(target->read_op_list_tail); + if (target->issued_read_op_list_head != NULL) { + op_list_head = &(target->issued_read_op_list_head); + op_list_tail = &(target->issued_read_op_list_tail); read_flag = 1; } - else if (target->write_op_list_head != NULL) { - op_list_head = &(target->write_op_list_head); - op_list_tail = &(target->write_op_list_tail); + else if (target->issued_write_op_list_head != NULL) { + op_list_head = &(target->issued_write_op_list_head); + op_list_tail = &(target->issued_write_op_list_tail); write_flag = 1; } - else if (target->dt_op_list_head != NULL) { - op_list_head = &(target->dt_op_list_head); - op_list_tail = &(target->dt_op_list_tail); + else if (target->issued_dt_op_list_head != NULL) { + op_list_head = &(target->issued_dt_op_list_head); + op_list_tail = &(target->issued_dt_op_list_tail); } else { /* only pending op list is not NULL, nothing we can do here. */ @@ -421,23 +421,23 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM if (*op_list_head == NULL) { if (read_flag == 1) { read_flag = 0; - if (target->write_op_list_head != NULL) { - op_list_head = &(target->write_op_list_head); - op_list_tail = &(target->write_op_list_tail); + if (target->issued_write_op_list_head != NULL) { + op_list_head = &(target->issued_write_op_list_head); + op_list_tail = &(target->issued_write_op_list_tail); write_flag = 1; } - else if (target->dt_op_list_head != NULL) { - op_list_head = &(target->dt_op_list_head); - op_list_tail = &(target->dt_op_list_tail); + else if (target->issued_dt_op_list_head != NULL) { + op_list_head = &(target->issued_dt_op_list_head); + op_list_tail = &(target->issued_dt_op_list_tail); } else break; } else if (write_flag == 1) { write_flag = 0; - if (target->dt_op_list_head != NULL) { - op_list_head = &(target->dt_op_list_head); - op_list_tail = &(target->dt_op_list_tail); + if (target->issued_dt_op_list_head != NULL) { + op_list_head = &(target->issued_dt_op_list_head); + op_list_tail = &(target->issued_dt_op_list_tail); } else break; @@ -452,8 +452,8 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM cleanup_target: if (target->pending_op_list_head == NULL && - target->read_op_list_head == NULL && target->write_op_list_head == NULL && - target->dt_op_list_head == NULL) { + target->issued_read_op_list_head == NULL && target->issued_write_op_list_head == NULL && + target->issued_dt_op_list_head == NULL) { (*local_completed) = 1; diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 1956488..0bfb49c 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -79,9 +79,9 @@ typedef struct MPIDI_RMA_Op { } MPIDI_RMA_Op_t; typedef struct MPIDI_RMA_Target { - struct MPIDI_RMA_Op *read_op_list_head, *read_op_list_tail; - struct MPIDI_RMA_Op *write_op_list_head, *write_op_list_tail; - struct MPIDI_RMA_Op *dt_op_list_head, *dt_op_list_tail; + struct MPIDI_RMA_Op *issued_read_op_list_head, *issued_read_op_list_tail; + struct MPIDI_RMA_Op *issued_write_op_list_head, *issued_write_op_list_tail; + struct MPIDI_RMA_Op *issued_dt_op_list_head, *issued_dt_op_list_tail; struct MPIDI_RMA_Op *pending_op_list_head, *pending_op_list_tail; struct MPIDI_RMA_Op *next_op_to_issue; struct MPIDI_RMA_Target *next; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 58e4c28..0d80984 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -658,19 +658,19 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, } if (is_derived) { - MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list_head), - &(target->dt_op_list_tail), op); + MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head), + &(target->issued_dt_op_list_tail), op); } else if (op->pkt.type == MPIDI_CH3_PKT_PUT || op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { - MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list_head), - &(target->write_op_list_tail), op); + MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head), + &(target->issued_write_op_list_tail), op); } else { - MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list_head), - &(target->read_op_list_tail), op); + MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head), + &(target->issued_read_op_list_tail), op); } } } diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index e94a171..1d1f395 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -429,19 +429,19 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ } if (is_derived) { - MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list_head), - &(target->dt_op_list_tail), curr_op); + MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head), + &(target->issued_dt_op_list_tail), curr_op); } else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { - MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list_head), - &(target->write_op_list_tail), curr_op); + MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head), + &(target->issued_write_op_list_tail), curr_op); } else { - MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list_head), - &(target->read_op_list_tail), curr_op); + MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head), + &(target->issued_read_op_list_tail), curr_op); } } @@ -549,8 +549,8 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) if (win_ptr->slots[i].target_list_head != NULL) { curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { - if (curr_target->read_op_list_head != NULL || - curr_target->write_op_list_head != NULL) { + if (curr_target->issued_read_op_list_head != NULL || + curr_target->issued_write_op_list_head != NULL) { if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET || win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) { if (curr_target->access_state == MPIDI_RMA_LOCK_GRANTED) @@ -574,14 +574,14 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) * must do a Win_flush instead. */ curr_target->disable_flush_local = 1; - if (curr_target->read_op_list_head != NULL) { - op_list_head = &curr_target->read_op_list_head; - op_list_tail = &curr_target->read_op_list_tail; + if (curr_target->issued_read_op_list_head != NULL) { + op_list_head = &curr_target->issued_read_op_list_head; + op_list_tail = &curr_target->issued_read_op_list_tail; read_flag = 1; } else { - op_list_head = &curr_target->write_op_list_head; - op_list_tail = &curr_target->write_op_list_tail; + op_list_head = &curr_target->issued_write_op_list_head; + op_list_tail = &curr_target->issued_write_op_list_tail; } /* free all ops in the list since we do not need to maintain them anymore */ @@ -606,8 +606,8 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) if (*op_list_head == NULL) { if (read_flag == 1) { - op_list_head = &curr_target->write_op_list_head; - op_list_head = &curr_target->write_op_list_tail; + op_list_head = &curr_target->issued_write_op_list_head; + op_list_head = &curr_target->issued_write_op_list_tail; read_flag = 0; } } http://git.mpich.org/mpich.git/commitdiff/2db95ef98d98d968af42eeea9d0b9b2a42... commit 2db95ef98d98d968af42eeea9d0b9b2a4293bfcc Author: Xin Zhao <[email protected]> Date: Mon Mar 9 20:09:49 2015 -0700 Add "_head" to name of linked lists. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index c644cdd..e5b3cff 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -111,10 +111,10 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr MPL_LL_DELETE(win_ptr->target_pool, win_ptr->target_pool_tail, e); } - e->read_op_list = e->read_op_list_tail = NULL; - e->write_op_list = e->write_op_list_tail = NULL; - e->dt_op_list = e->dt_op_list_tail = NULL; - e->pending_op_list = e->pending_op_list_tail = NULL; + e->read_op_list_head = e->read_op_list_tail = NULL; + e->write_op_list_head = e->write_op_list_tail = NULL; + e->dt_op_list_head = e->dt_op_list_tail = NULL; + e->pending_op_list_head = e->pending_op_list_tail = NULL; e->next_op_to_issue = NULL; e->target_rank = -1; @@ -145,10 +145,10 @@ static inline int MPIDI_CH3I_Win_target_free(MPID_Win * win_ptr, MPIDI_RMA_Targe /* We enqueue elements to the right pool, so when they get freed * at window free time, they won't conflict with the global pool * or other windows */ - MPIU_Assert(e->read_op_list == NULL); - MPIU_Assert(e->write_op_list == NULL); - MPIU_Assert(e->dt_op_list == NULL); - MPIU_Assert(e->pending_op_list == NULL); + MPIU_Assert(e->read_op_list_head == NULL); + MPIU_Assert(e->write_op_list_head == NULL); + MPIU_Assert(e->dt_op_list_head == NULL); + MPIU_Assert(e->pending_op_list_head == NULL); /* use PREPEND when return objects back to the pool * in order to improve cache performance */ @@ -187,11 +187,11 @@ static inline int MPIDI_CH3I_Win_create_target(MPID_Win * win_ptr, int target_ra t->target_rank = target_rank; - if (slot->target_list == NULL) + if (slot->target_list_head == NULL) win_ptr->non_empty_slots++; /* Enqueue target into target list. */ - MPL_LL_APPEND(slot->target_list, slot->target_list_tail, t); + MPL_LL_APPEND(slot->target_list_head, slot->target_list_tail, t); assert(t != NULL); @@ -221,7 +221,7 @@ static inline int MPIDI_CH3I_Win_find_target(MPID_Win * win_ptr, int target_rank else slot = &(win_ptr->slots[target_rank]); - t = slot->target_list; + t = slot->target_list_head; while (t != NULL) { if (t->target_rank == target_rank) break; @@ -277,7 +277,7 @@ static inline int MPIDI_CH3I_Win_enqueue_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t * } /* Enqueue operation into pending list. */ - MPL_LL_APPEND(target->pending_op_list, target->pending_op_list_tail, op); + MPL_LL_APPEND(target->pending_op_list_head, target->pending_op_list_tail, op); if (target->next_op_to_issue == NULL) target->next_op_to_issue = op; @@ -309,13 +309,13 @@ static inline int MPIDI_CH3I_Win_target_dequeue_and_free(MPID_Win * win_ptr, MPI else slot = &(win_ptr->slots[target_rank]); - MPL_LL_DELETE(slot->target_list, slot->target_list_tail, e); + MPL_LL_DELETE(slot->target_list_head, slot->target_list_tail, e); mpi_errno = MPIDI_CH3I_Win_target_free(win_ptr, e); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (slot->target_list == NULL) + if (slot->target_list_head == NULL) win_ptr->non_empty_slots--; fn_exit: @@ -333,7 +333,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM int *local_completed, int *remote_completed) { MPIDI_RMA_Op_t *curr_op = NULL; - MPIDI_RMA_Op_t **op_list = NULL, **op_list_tail = NULL; + MPIDI_RMA_Op_t **op_list_head = NULL, **op_list_tail = NULL; int read_flag = 0, write_flag = 0; int mpi_errno = MPI_SUCCESS; int i; @@ -353,22 +353,23 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM target->access_state == MPIDI_RMA_LOCK_ISSUED) goto fn_exit; - if (target->pending_op_list == NULL && - target->read_op_list == NULL && target->write_op_list == NULL && target->dt_op_list == NULL) + if (target->pending_op_list_head == NULL && + target->read_op_list_head == NULL && target->write_op_list_head == NULL && + target->dt_op_list_head == NULL) goto cleanup_target; - if (target->read_op_list != NULL) { - op_list = &(target->read_op_list); + if (target->read_op_list_head != NULL) { + op_list_head = &(target->read_op_list_head); op_list_tail = &(target->read_op_list_tail); read_flag = 1; } - else if (target->write_op_list != NULL) { - op_list = &(target->write_op_list); + else if (target->write_op_list_head != NULL) { + op_list_head = &(target->write_op_list_head); op_list_tail = &(target->write_op_list_tail); write_flag = 1; } - else if (target->dt_op_list != NULL) { - op_list = &(target->dt_op_list); + else if (target->dt_op_list_head != NULL) { + op_list_head = &(target->dt_op_list_head); op_list_tail = &(target->dt_op_list_tail); } else { @@ -376,7 +377,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM goto fn_exit; } - curr_op = *op_list; + curr_op = *op_list_head; while (curr_op != NULL) { for (i = 0; i < curr_op->reqs_size; i++) { if (curr_op->reqs[i] == NULL) @@ -414,19 +415,19 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM curr_op->reqs_size = 0; /* dequeue the operation and free it */ - MPL_LL_DELETE(*op_list, *op_list_tail, curr_op); + MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op); MPIDI_CH3I_Win_op_free(win_ptr, curr_op); - if (*op_list == NULL) { + if (*op_list_head == NULL) { if (read_flag == 1) { read_flag = 0; - if (target->write_op_list != NULL) { - op_list = &(target->write_op_list); + if (target->write_op_list_head != NULL) { + op_list_head = &(target->write_op_list_head); op_list_tail = &(target->write_op_list_tail); write_flag = 1; } - else if (target->dt_op_list != NULL) { - op_list = &(target->dt_op_list); + else if (target->dt_op_list_head != NULL) { + op_list_head = &(target->dt_op_list_head); op_list_tail = &(target->dt_op_list_tail); } else @@ -434,8 +435,8 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM } else if (write_flag == 1) { write_flag = 0; - if (target->dt_op_list != NULL) { - op_list = &(target->dt_op_list); + if (target->dt_op_list_head != NULL) { + op_list_head = &(target->dt_op_list_head); op_list_tail = &(target->dt_op_list_tail); } else @@ -443,16 +444,16 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM } } /* next op */ - curr_op = *op_list; + curr_op = *op_list_head; } else break; } cleanup_target: - if (target->pending_op_list == NULL && - target->read_op_list == NULL && target->write_op_list == NULL && - target->dt_op_list == NULL) { + if (target->pending_op_list_head == NULL && + target->read_op_list_head == NULL && target->write_op_list_head == NULL && + target->dt_op_list_head == NULL) { (*local_completed) = 1; @@ -486,7 +487,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_win(MPID_Win * win_ptr, (*remote_completed) = 0; for (i = 0; i < win_ptr->num_slots; i++) { - for (target = win_ptr->slots[i].target_list; target;) { + for (target = win_ptr->slots[i].target_list_head; target;) { int local = 0, remote = 0; mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target, &local, &remote); @@ -544,7 +545,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_targets_win(MPID_Win * win_ptr) int i, mpi_errno = MPI_SUCCESS; for (i = 0; i < win_ptr->num_slots; i++) { - for (target = win_ptr->slots[i].target_list; target;) { + for (target = win_ptr->slots[i].target_list_head; target;) { next_target = target->next; mpi_errno = MPIDI_CH3I_RMA_Cleanup_single_target(win_ptr, target); if (mpi_errno != MPI_SUCCESS) diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 732e859..1956488 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -79,10 +79,10 @@ typedef struct MPIDI_RMA_Op { } MPIDI_RMA_Op_t; typedef struct MPIDI_RMA_Target { - struct MPIDI_RMA_Op *read_op_list, *read_op_list_tail; - struct MPIDI_RMA_Op *write_op_list, *write_op_list_tail; - struct MPIDI_RMA_Op *dt_op_list, *dt_op_list_tail; - struct MPIDI_RMA_Op *pending_op_list, *pending_op_list_tail; + struct MPIDI_RMA_Op *read_op_list_head, *read_op_list_tail; + struct MPIDI_RMA_Op *write_op_list_head, *write_op_list_tail; + struct MPIDI_RMA_Op *dt_op_list_head, *dt_op_list_tail; + struct MPIDI_RMA_Op *pending_op_list_head, *pending_op_list_tail; struct MPIDI_RMA_Op *next_op_to_issue; struct MPIDI_RMA_Target *next; int target_rank; @@ -114,7 +114,7 @@ typedef struct MPIDI_RMA_Target { } MPIDI_RMA_Target_t; typedef struct MPIDI_RMA_Slot { - struct MPIDI_RMA_Target *target_list; + struct MPIDI_RMA_Target *target_list_head; struct MPIDI_RMA_Target *target_list_tail; } MPIDI_RMA_Slot_t; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 2eb7f70..58e4c28 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -619,7 +619,7 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, MPIU_ERR_POP(mpi_errno); MPIU_Assert(target != NULL); - op = target->pending_op_list; + op = target->pending_op_list_head; if (op != NULL) MPIDI_CH3_PKT_RMA_GET_FLAGS(op->pkt, op_flags, mpi_errno); @@ -636,14 +636,14 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, if (op->reqs_size == 0) { MPIU_Assert(op->reqs == NULL); - MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list), + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head), &(target->pending_op_list_tail), op); } else { MPI_Datatype target_datatype; int is_derived = FALSE; - MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list), + MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head), &(target->pending_op_list_tail), op); MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(op->pkt, target_datatype, mpi_errno); @@ -658,18 +658,18 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, } if (is_derived) { - MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list), + MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list_head), &(target->dt_op_list_tail), op); } else if (op->pkt.type == MPIDI_CH3_PKT_PUT || op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { - MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list), + MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list_head), &(target->write_op_list_tail), op); } else { - MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list), + MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list_head), &(target->read_op_list_tail), op); } } diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 825452c..e94a171 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -159,8 +159,8 @@ static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * ta if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) { - if (target->pending_op_list == NULL || - !target->pending_op_list->piggyback_lock_candidate) { + if (target->pending_op_list_head == NULL || + !target->pending_op_list_head->piggyback_lock_candidate) { /* issue lock request */ target->access_state = MPIDI_RMA_LOCK_ISSUED; if (target->target_rank == rank) { @@ -178,7 +178,7 @@ static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * ta } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) { - if (target->pending_op_list == NULL) { + if (target->pending_op_list_head == NULL) { /* No RMA operation has ever been posted to this target, * finish issuing, no need to acquire the lock. Cleanup * function will clean it up. */ @@ -196,8 +196,8 @@ static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * ta /* if we reach WIN_UNLOCK and there is still operation existing * in pending list, this operation must be the only operation * and it is prepared to piggyback LOCK and UNLOCK. */ - MPIU_Assert(target->pending_op_list->next == NULL); - MPIU_Assert(target->pending_op_list->piggyback_lock_candidate); + MPIU_Assert(target->pending_op_list_head->next == NULL); + MPIU_Assert(target->pending_op_list_head->piggyback_lock_candidate); } } break; @@ -205,7 +205,7 @@ static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * ta case MPIDI_RMA_LOCK_GRANTED: case MPIDI_RMA_NONE: if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) { - if (target->pending_op_list == NULL) { + if (target->pending_op_list_head == NULL) { if (target->target_rank == rank) { target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); @@ -232,7 +232,7 @@ static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * ta } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) { - if (target->pending_op_list == NULL) { + if (target->pending_op_list_head == NULL) { if (target->target_rank == rank) { target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); @@ -405,7 +405,7 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ if (curr_op->reqs_size == 0) { MPIU_Assert(curr_op->reqs == NULL); /* Sending is completed immediately. */ - MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list), + MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head), &(target->pending_op_list_tail), curr_op); } else { @@ -414,7 +414,7 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ /* Sending is not completed immediately. */ - MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list), + MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head), &(target->pending_op_list_tail), curr_op); MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno); @@ -429,18 +429,18 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ } if (is_derived) { - MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list), + MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list_head), &(target->dt_op_list_tail), curr_op); } else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { - MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list), + MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list_head), &(target->write_op_list_tail), curr_op); } else { - MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list), + MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list_head), &(target->read_op_list_tail), curr_op); } } @@ -487,7 +487,7 @@ static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress) else idx = i - win_ptr->num_slots; - target = win_ptr->slots[idx].target_list; + target = win_ptr->slots[idx].target_list_head; while (target != NULL) { int temp_progress = 0; @@ -524,7 +524,7 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) { MPIDI_RMA_Op_t *curr_op = NULL; MPIDI_RMA_Target_t *curr_target = NULL; - MPIDI_RMA_Op_t **op_list = NULL, **op_list_tail = NULL; + MPIDI_RMA_Op_t **op_list_head = NULL, **op_list_tail = NULL; int read_flag = 0; int i, made_progress = 0; int mpi_errno = MPI_SUCCESS; @@ -546,10 +546,11 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) /* find targets that have operations */ for (i = 0; i < win_ptr->num_slots; i++) { - if (win_ptr->slots[i].target_list != NULL) { - curr_target = win_ptr->slots[i].target_list; + if (win_ptr->slots[i].target_list_head != NULL) { + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { - if (curr_target->read_op_list != NULL || curr_target->write_op_list != NULL) { + if (curr_target->read_op_list_head != NULL || + curr_target->write_op_list_head != NULL) { if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET || win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) { if (curr_target->access_state == MPIDI_RMA_LOCK_GRANTED) @@ -573,18 +574,18 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) * must do a Win_flush instead. */ curr_target->disable_flush_local = 1; - if (curr_target->read_op_list != NULL) { - op_list = &curr_target->read_op_list; + if (curr_target->read_op_list_head != NULL) { + op_list_head = &curr_target->read_op_list_head; op_list_tail = &curr_target->read_op_list_tail; read_flag = 1; } else { - op_list = &curr_target->write_op_list; + op_list_head = &curr_target->write_op_list_head; op_list_tail = &curr_target->write_op_list_tail; } /* free all ops in the list since we do not need to maintain them anymore */ - for (curr_op = *op_list; curr_op != NULL;) { + for (curr_op = *op_list_head; curr_op != NULL;) { if (curr_op->reqs_size > 0) { MPIU_Assert(curr_op->reqs != NULL); for (i = 0; i < curr_op->reqs_size; i++) { @@ -600,17 +601,17 @@ int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr) curr_op->reqs = NULL; curr_op->reqs_size = 0; } - MPL_LL_DELETE(*op_list, *op_list_tail, curr_op); + MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op); MPIDI_CH3I_Win_op_free(win_ptr, curr_op); - if (*op_list == NULL) { + if (*op_list_head == NULL) { if (read_flag == 1) { - op_list = &curr_target->write_op_list; - op_list = &curr_target->write_op_list_tail; + op_list_head = &curr_target->write_op_list_head; + op_list_head = &curr_target->write_op_list_tail; read_flag = 0; } } - curr_op = *op_list; + curr_op = *op_list_head; } fn_exit: @@ -638,9 +639,9 @@ int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr) /* find the first target that has something to issue */ for (i = 0; i < win_ptr->num_slots; i++) { - if (win_ptr->slots[i].target_list != NULL) { - curr_target = win_ptr->slots[i].target_list; - while (curr_target != NULL && curr_target->pending_op_list == NULL) + if (win_ptr->slots[i].target_list_head != NULL) { + curr_target = win_ptr->slots[i].target_list_head; + while (curr_target != NULL && curr_target->pending_op_list_head == NULL) curr_target = curr_target->next; if (curr_target != NULL) break; @@ -725,9 +726,9 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe * target */ /* TODO: we should think about better strategies on selecting the target */ for (i = 0; i < win_ptr->num_slots; i++) - if (win_ptr->slots[i].target_list != NULL) + if (win_ptr->slots[i].target_list_head != NULL) break; - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; curr_target->sync.outstanding_acks++; diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 9dad49a..0e06bc8 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -368,7 +368,7 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) /* Set sync_flag in target structs. */ for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { /* set sync_flag in sync struct */ @@ -742,9 +742,9 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) } if (win_comm_ptr->local_size <= win_ptr->num_slots) - curr_target = win_ptr->slots[dst].target_list; + curr_target = win_ptr->slots[dst].target_list_head; else { - curr_target = win_ptr->slots[dst % win_ptr->num_slots].target_list; + curr_target = win_ptr->slots[dst % win_ptr->num_slots].target_list_head; while (curr_target != NULL && curr_target->target_rank != dst) curr_target = curr_target->next; } @@ -1522,7 +1522,7 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr) if (win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) { for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { if (curr_target->sync.sync_flag < sync_flag) { curr_target->sync.sync_flag = sync_flag; @@ -1535,9 +1535,9 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr) else { for (i = 0; i < win_ptr->comm_ptr->local_size; i++) { if (win_ptr->comm_ptr->local_size <= win_ptr->num_slots) - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; else { - curr_target = win_ptr->slots[i % win_ptr->num_slots].target_list; + curr_target = win_ptr->slots[i % win_ptr->num_slots].target_list_head; while (curr_target != NULL && curr_target->target_rank != i) curr_target = curr_target->next; } @@ -1662,7 +1662,7 @@ int MPIDI_Win_flush_all(MPID_Win * win_ptr) /* Set sync_flag in sync struct. */ for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; @@ -1756,7 +1756,7 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) /* Set sync_flag in sync struct. */ for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { if (curr_target->disable_flush_local) { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { @@ -1788,7 +1788,7 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) * and wait for local completion for other targets */ do { for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target, &local_completed, &remote_completed); @@ -1825,7 +1825,7 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) finish_flush_local_all: /* reset disable_flush_local flag in target to 0 */ for (i = 0; i < win_ptr->num_slots; i++) { - curr_target = win_ptr->slots[i].target_list; + curr_target = win_ptr->slots[i].target_list_head; while (curr_target != NULL) { curr_target->disable_flush_local = 0; curr_target = curr_target->next; diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index 5e86d3a..0c3892e 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -371,7 +371,7 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, MPIU_CHKPMEM_MALLOC((*win_ptr)->slots, MPIDI_RMA_Slot_t *, sizeof(MPIDI_RMA_Slot_t) * (*win_ptr)->num_slots, mpi_errno, "RMA slots"); for (i = 0; i < (*win_ptr)->num_slots; i++) { - (*win_ptr)->slots[i].target_list = NULL; + (*win_ptr)->slots[i].target_list_head = NULL; (*win_ptr)->slots[i].target_list_tail = NULL; } http://git.mpich.org/mpich.git/commitdiff/299ede6e194bb0f7329511c83c3d0bf671... commit 299ede6e194bb0f7329511c83c3d0bf671ced860 Author: Xin Zhao <[email protected]> Date: Mon Mar 9 16:12:37 2015 -0700 Remove "is_dt" flag from op structure. Originally we use is_dt flag stored in op structure to track if derived datatypes are involved in the current RMA operation. The flag is actually not needed and we can directly check datatypes in the current operation. Here we remove is_dt flag from the op structure. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index a0c513a..c644cdd 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -50,10 +50,12 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr) e->reqs = NULL; e->reqs_size = 0; e->ureq = NULL; - e->is_dt = 0; e->piggyback_lock_candidate = 0; e->issued_stream_count = 0; + e->origin_datatype = MPI_DATATYPE_NULL; + e->result_datatype = MPI_DATATYPE_NULL; + return e; } diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 6d8f477..732e859 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -70,7 +70,6 @@ typedef struct MPIDI_RMA_Op { MPIDI_CH3_Pkt_t pkt; MPIDI_RMA_Pool_type_t pool_type; - int is_dt; int piggyback_lock_candidate; int issued_stream_count; /* when >= 0, it specifies number of stream units that have been issued; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 3ddb28e..2eb7f70 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -640,9 +640,24 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr, &(target->pending_op_list_tail), op); } else { + MPI_Datatype target_datatype; + int is_derived = FALSE; + MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list), &(target->pending_op_list_tail), op); - if (op->is_dt) { + + MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(op->pkt, target_datatype, mpi_errno); + + if ((target_datatype != MPI_DATATYPE_NULL && + !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) || + (op->origin_datatype != MPI_DATATYPE_NULL && + !MPIR_DATATYPE_IS_PREDEFINED(op->origin_datatype)) || + (op->result_datatype != MPI_DATATYPE_NULL && + !MPIR_DATATYPE_IS_PREDEFINED(op->result_datatype))) { + is_derived = TRUE; + } + + if (is_derived) { MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list), &(target->dt_op_list_tail), op); } diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 9e9c70a..a8fd127 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -145,12 +145,10 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, dtp); MPID_Datatype_add_ref(dtp); - op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); - op_ptr->is_dt = 1; } MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); @@ -160,13 +158,15 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); /* Judge if we can use IMMED data packet */ - if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } /* Judge if this operation is an piggyback candidate */ - if (!op_ptr->is_dt) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ @@ -336,12 +336,10 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, dtp); MPID_Datatype_add_ref(dtp); - op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); - op_ptr->is_dt = 1; } MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); @@ -351,13 +349,15 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIU_Assign_trunc(len, target_count * target_type_size, size_t); /* Judge if we can use IMMED data response packet */ - if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } /* Judge if this operation is an piggyback candidate. */ - if (!op_ptr->is_dt) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ @@ -518,11 +518,9 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, origin_dtp); - op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, target_dtp); - op_ptr->is_dt = 1; } MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); @@ -562,13 +560,15 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPID_Datatype_is_contig(target_datatype, &is_target_contig); /* Judge if we can use IMMED data packet */ - if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } /* Judge if this operation is an piggyback candidate. */ - if (!op_ptr->is_dt) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ @@ -746,12 +746,10 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, dtp); MPID_Datatype_add_ref(dtp); - op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); - op_ptr->is_dt = 1; } MPID_Datatype_get_size_macro(target_datatype, target_type_size); @@ -761,13 +759,16 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_is_contig(target_datatype, &is_target_contig); /* Judge if we can use IMMED data response packet */ - if (!op_ptr->is_dt && is_result_contig && is_target_contig) { + if (MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && + is_result_contig && is_target_contig) { if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } /* Judge if this operation is a piggyback candidate */ - if (!op_ptr->is_dt) { + if (MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ @@ -817,15 +818,12 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, origin_dtp); - op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, target_dtp); - op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, result_dtp); - op_ptr->is_dt = 1; } MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); @@ -869,13 +867,18 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_is_contig(result_datatype, &is_result_contig); /* Judge if we can use IMMED data packet */ - if (!op_ptr->is_dt && is_origin_contig && is_target_contig && is_result_contig) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && + is_origin_contig && is_target_contig && is_result_contig) { if (orig_len <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } /* Judge if this operation is a piggyback candidate */ - if (!op_ptr->is_dt) { + if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(result_datatype) && + MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for origin, target and result data. We should extend this optimization to derived * datatypes as well. */ diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index 901c190..825452c 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -409,11 +409,26 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ &(target->pending_op_list_tail), curr_op); } else { + MPI_Datatype target_datatype; + int is_derived = FALSE; + /* Sending is not completed immediately. */ MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list), &(target->pending_op_list_tail), curr_op); - if (curr_op->is_dt) { + + MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno); + + if ((target_datatype != MPI_DATATYPE_NULL && + !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) || + (curr_op->origin_datatype != MPI_DATATYPE_NULL && + !MPIR_DATATYPE_IS_PREDEFINED(curr_op->origin_datatype)) || + (curr_op->result_datatype != MPI_DATATYPE_NULL && + !MPIR_DATATYPE_IS_PREDEFINED(curr_op->result_datatype))) { + is_derived = TRUE; + } + + if (is_derived) { MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list), &(target->dt_op_list_tail), curr_op); } http://git.mpich.org/mpich.git/commitdiff/35a05b3ca1e0cc60b1d1d607f408e0ab1e... commit 35a05b3ca1e0cc60b1d1d607f408e0ab1e1f9ced Author: Xin Zhao <[email protected]> Date: Mon Mar 9 15:33:42 2015 -0700 Remove unused have_remote_incomplete_ops. have_remote_incomplete_ops is not actually used in the code, we remove it here. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_oplist.h b/src/mpid/ch3/include/mpid_rma_oplist.h index e04a742..a0c513a 100644 --- a/src/mpid/ch3/include/mpid_rma_oplist.h +++ b/src/mpid/ch3/include/mpid_rma_oplist.h @@ -126,9 +126,7 @@ static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr e->sync.sync_flag = MPIDI_RMA_SYNC_NONE; e->sync.outstanding_acks = 0; - e->sync.have_remote_incomplete_ops = 1; /* When I create a new target, there must be - * incomplete ops until a FLUSH/UNLOCK packet - * is sent. */ + return e; } @@ -459,8 +457,7 @@ static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RM /* for the conditions that need to be satisfied before we free the * target, see the MPIDI_RMA_Target definition in * mpid_rma_types.h */ - if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && - target->sync.outstanding_acks == 0 && target->sync.have_remote_incomplete_ops == 0) { + if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && target->sync.outstanding_acks == 0) { (*remote_completed) = 1; } } diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index be0e869..6d8f477 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -100,7 +100,6 @@ typedef struct MPIDI_RMA_Target { * following conditions hold true: * - No operations are queued up (op_list == NULL) * - There are no outstanding acks (outstanding_acks == 0) - * - There are no incomplete ops (have_remote_incomplete_ops == 0) * - There are no sync messages to be sent (sync_flag == NONE) */ struct { @@ -110,12 +109,6 @@ typedef struct MPIDI_RMA_Target { /* packets sent out that we are expecting an ack for */ int outstanding_acks; - - /* if we sent out any operations, but have not waited for - * their remote completion, this flag is set. When the next - * FLUSH or UNLOCK sync flag is set, we will clear this - * variable. */ - int have_remote_incomplete_ops; /* have ops that have not completed remotely */ } sync; MPIDI_RMA_Pool_type_t pool_type; diff --git a/src/mpid/ch3/src/ch3u_rma_progress.c b/src/mpid/ch3/src/ch3u_rma_progress.c index de3c0d2..901c190 100644 --- a/src/mpid/ch3/src/ch3u_rma_progress.c +++ b/src/mpid/ch3/src/ch3u_rma_progress.c @@ -715,7 +715,6 @@ int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Targe curr_target = win_ptr->slots[i].target_list; if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 94de1c1..9dad49a 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -374,7 +374,6 @@ int MPIDI_Win_fence(int assert, MPID_Win * win_ptr) /* set sync_flag in sync struct */ if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } curr_target = curr_target->next; @@ -754,7 +753,6 @@ int MPIDI_Win_complete(MPID_Win * win_ptr) /* set sync_flag in sync struct */ if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } curr_target->win_complete_flag = 1; @@ -1090,7 +1088,6 @@ int MPIDI_Win_unlock(int dest, MPID_Win * win_ptr) sync_flag = MPIDI_RMA_SYNC_UNLOCK; if (target->sync.sync_flag < sync_flag) { target->sync.sync_flag = sync_flag; - target->sync.have_remote_incomplete_ops = 0; target->sync.outstanding_acks++; } @@ -1211,7 +1208,6 @@ int MPIDI_Win_flush(int dest, MPID_Win * win_ptr) /* Set sync_flag in sync struct. */ if (target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - target->sync.have_remote_incomplete_ops = 0; target->sync.outstanding_acks++; } @@ -1318,7 +1314,6 @@ int MPIDI_Win_flush_local(int dest, MPID_Win * win_ptr) if (target->disable_flush_local) { if (target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - target->sync.have_remote_incomplete_ops = 0; target->sync.outstanding_acks++; } } @@ -1531,7 +1526,6 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr) while (curr_target != NULL) { if (curr_target->sync.sync_flag < sync_flag) { curr_target->sync.sync_flag = sync_flag; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } curr_target = curr_target->next; @@ -1551,7 +1545,6 @@ int MPIDI_Win_unlock_all(MPID_Win * win_ptr) if (curr_target != NULL) { if (curr_target->sync.sync_flag < sync_flag) { curr_target->sync.sync_flag = sync_flag; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } } @@ -1673,7 +1666,6 @@ int MPIDI_Win_flush_all(MPID_Win * win_ptr) while (curr_target != NULL) { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } @@ -1769,7 +1761,6 @@ int MPIDI_Win_flush_local_all(MPID_Win * win_ptr) if (curr_target->disable_flush_local) { if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) { curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH; - curr_target->sync.have_remote_incomplete_ops = 0; curr_target->sync.outstanding_acks++; } disable_flush_local_cnt++; http://git.mpich.org/mpich.git/commitdiff/98c791a6cfbc87698d4304ebdcb994b4c2... commit 98c791a6cfbc87698d4304ebdcb994b4c212fba1 Author: Xin Zhao <[email protected]> Date: Mon Mar 9 15:30:26 2015 -0700 Simplify code: delete unncessary calculation in IMMED length. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index f1d9725..9e9c70a 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -118,7 +118,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_put_t *put_pkt = NULL; MPI_Aint origin_type_size; - size_t immed_len, len; + size_t len; int use_immed_pkt = FALSE; int is_origin_contig, is_target_contig; @@ -161,10 +161,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype /* Judge if we can use IMMED data packet */ if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size, - size_t); - if (len <= immed_len) + if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } @@ -312,7 +309,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; MPI_Aint target_type_size; - size_t immed_len, len; + size_t len; int use_immed_resp_pkt = FALSE; int is_origin_contig, is_target_contig; @@ -355,10 +352,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype /* Judge if we can use IMMED data response packet */ if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size, - size_t); - if (len <= immed_len) + if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } @@ -495,7 +489,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; MPI_Aint origin_type_size; - size_t immed_len, len; + size_t len; int use_immed_pkt = FALSE; int is_origin_contig, is_target_contig; MPI_Aint stream_elem_count, stream_unit_count; @@ -569,10 +563,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp /* Judge if we can use IMMED data packet */ if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size, - size_t); - if (len <= immed_len) + if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } @@ -738,7 +729,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /* Convert GAcc to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; MPI_Aint target_type_size; - size_t len, immed_len; + size_t len; int use_immed_resp_pkt = FALSE; int is_result_contig, is_target_contig; @@ -771,10 +762,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /* Judge if we can use IMMED data response packet */ if (!op_ptr->is_dt && is_result_contig && is_target_contig) { - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size, - size_t); - if (len <= immed_len) + if (len <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } @@ -804,7 +792,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, else { MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; MPI_Aint origin_type_size; - size_t immed_len, orig_len; + size_t orig_len; int use_immed_pkt = FALSE; int is_origin_contig, is_target_contig, is_result_contig; MPI_Aint stream_elem_count, stream_unit_count; @@ -882,10 +870,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /* Judge if we can use IMMED data packet */ if (!op_ptr->is_dt && is_origin_contig && is_target_contig && is_result_contig) { - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size, - size_t); - if (orig_len <= immed_len) + if (orig_len <= MPIDI_RMA_IMMED_BYTES) use_immed_pkt = TRUE; } @@ -1293,7 +1278,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /* Convert FOP to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; MPI_Aint target_type_size; - size_t immed_len; int use_immed_resp_pkt = FALSE; int is_contig; @@ -1314,10 +1298,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, if (is_contig) { /* Judege if we can use IMMED data for response packet */ - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size, - size_t); - if (target_type_size <= immed_len) + if (target_type_size <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } @@ -1336,7 +1317,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, else { MPIDI_CH3_Pkt_fop_t *fop_pkt; MPI_Aint type_size; - size_t immed_len; int use_immed_pkt = FALSE; int is_contig; @@ -1359,9 +1339,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, if (is_contig) { /* Judge if we can use IMMED data packet */ - MPIU_Assign_trunc(immed_len, - (MPIDI_RMA_IMMED_BYTES / type_size) * type_size, size_t); - if (type_size <= immed_len) { + if (type_size <= MPIDI_RMA_IMMED_BYTES) { use_immed_pkt = TRUE; } } http://git.mpich.org/mpich.git/commitdiff/6ee07af463b5de7fd11e9b94fc18ed5cd5... commit 6ee07af463b5de7fd11e9b94fc18ed5cd595a4dc Author: Xin Zhao <[email protected]> Date: Mon Mar 9 15:23:27 2015 -0700 Simplify code: use MPIDI_CH3U_Request_complete to complete user request. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index f530a24..6069ffc 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -1298,8 +1298,7 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op) * and release ch3 ref. */ /* Complete user request and release the ch3 ref */ - MPID_Request_set_completed(op->ureq); - MPID_Request_release(op->ureq); + MPIDI_CH3U_Request_complete(op->ureq); } else { /* Sending is not completed immediately. */ @@ -1344,8 +1343,7 @@ static inline int set_user_req_after_issuing_op(MPIDI_RMA_Op_t * op) else { /* all requests are completed */ /* Complete user request and release ch3 ref */ - MPID_Request_set_completed(op->ureq); - MPID_Request_release(op->ureq); + MPIDI_CH3U_Request_complete(op->ureq); op->ureq = NULL; } } diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index ded1f1a..f1d9725 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -111,8 +111,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype if (ureq) { /* Complete user request and release the ch3 ref */ - MPID_Request_set_completed(ureq); - MPID_Request_release(ureq); + MPIDI_CH3U_Request_complete(ureq); } } else { @@ -306,8 +305,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype if (ureq) { /* Complete user request and release the ch3 ref */ - MPID_Request_set_completed(ureq); - MPID_Request_release(ureq); + MPIDI_CH3U_Request_complete(ureq); } } else { @@ -490,8 +488,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp if (ureq) { /* Complete user request and release the ch3 ref */ - MPID_Request_set_completed(ureq); - MPID_Request_release(ureq); + MPIDI_CH3U_Request_complete(ureq); } } else { @@ -722,8 +719,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (ureq) { /* Complete user request and release the ch3 ref */ - MPID_Request_set_completed(ureq); - MPID_Request_release(ureq); + MPIDI_CH3U_Request_complete(ureq); } } else { http://git.mpich.org/mpich.git/commitdiff/6394c667ff22fed27bf0765c558e787a49... commit 6394c667ff22fed27bf0765c558e787a49ac66f8 Author: Xin Zhao <[email protected]> Date: Mon Mar 9 15:19:34 2015 -0700 Delete useless IF branch. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 3a78163..ded1f1a 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -139,9 +139,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype op_ptr->target_rank = target_rank; /* Remember user request */ - if (ureq) { - op_ptr->ureq = ureq; - } + op_ptr->ureq = ureq; /* if source or target datatypes are derived, increment their * reference counts */ @@ -336,9 +334,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype op_ptr->target_rank = target_rank; /* Remember user request */ - if (ureq) { - op_ptr->ureq = ureq; - } + op_ptr->ureq = ureq; /* if source or target datatypes are derived, increment their * reference counts */ @@ -525,9 +521,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp op_ptr->target_rank = target_rank; /* Remember user request */ - if (ureq) { - op_ptr->ureq = ureq; - } + op_ptr->ureq = ureq; /* if source or target datatypes are derived, increment their * reference counts */ @@ -760,9 +754,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, op_ptr->target_rank = target_rank; /* Remember user request */ - if (ureq) { - op_ptr->ureq = ureq; - } + op_ptr->ureq = ureq; if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, dtp); @@ -835,9 +827,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, op_ptr->target_rank = target_rank; /* Remember user request */ - if (ureq) { - op_ptr->ureq = ureq; - } + op_ptr->ureq = ureq; /* if source or target datatypes are derived, increment their * reference counts */ http://git.mpich.org/mpich.git/commitdiff/86bb183473bb67c63a22c1b6e16d96d885... commit 86bb183473bb67c63a22c1b6e16d96d88581159f Author: Xin Zhao <[email protected]> Date: Fri May 15 14:33:01 2015 -0500 Change op pointer name 'new_ptr' to 'op_ptr'. Originally 'new_ptr' is used for pointer to RMA operation. Here we change it from 'new_ptr' to 'op_ptr' to make the name clearer. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 424ad1e..3a78163 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -116,7 +116,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype } } else { - MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_put_t *put_pkt = NULL; MPI_Aint origin_type_size; size_t immed_len, len; @@ -124,7 +124,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype int is_origin_contig, is_target_contig; /* queue it up */ - mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); + mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -133,14 +133,14 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = origin_count; - new_ptr->origin_datatype = origin_datatype; - new_ptr->target_rank = target_rank; + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = origin_count; + op_ptr->origin_datatype = origin_datatype; + op_ptr->target_rank = target_rank; /* Remember user request */ if (ureq) { - new_ptr->ureq = ureq; + op_ptr->ureq = ureq; } /* if source or target datatypes are derived, increment their @@ -148,12 +148,12 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, dtp); MPID_Datatype_add_ref(dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); @@ -163,7 +163,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); /* Judge if we can use IMMED data packet */ - if (!new_ptr->is_dt && is_origin_contig && is_target_contig) { + if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { MPIU_Assign_trunc(immed_len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size, size_t); @@ -172,17 +172,17 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype } /* Judge if this operation is an piggyback candidate */ - if (!new_ptr->is_dt) { + if (!op_ptr->is_dt) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) - new_ptr->piggyback_lock_candidate = 1; + op_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ - put_pkt = &(new_ptr->pkt.put); + put_pkt = &(op_ptr->pkt.put); if (use_immed_pkt) { MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT_IMMED); @@ -208,7 +208,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -313,7 +313,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype } } else { - MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; MPI_Aint target_type_size; size_t immed_len, len; @@ -321,7 +321,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype int is_origin_contig, is_target_contig; /* queue it up */ - mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); + mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -330,14 +330,14 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ - new_ptr->origin_addr = origin_addr; - new_ptr->origin_count = origin_count; - new_ptr->origin_datatype = origin_datatype; - new_ptr->target_rank = target_rank; + op_ptr->origin_addr = origin_addr; + op_ptr->origin_count = origin_count; + op_ptr->origin_datatype = origin_datatype; + op_ptr->target_rank = target_rank; /* Remember user request */ if (ureq) { - new_ptr->ureq = ureq; + op_ptr->ureq = ureq; } /* if source or target datatypes are derived, increment their @@ -345,12 +345,12 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, dtp); MPID_Datatype_add_ref(dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); @@ -360,7 +360,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIU_Assign_trunc(len, target_count * target_type_size, size_t); /* Judge if we can use IMMED data response packet */ - if (!new_ptr->is_dt && is_origin_contig && is_target_contig) { + if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { MPIU_Assign_trunc(immed_len, (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size, size_t); @@ -369,16 +369,16 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype } /* Judge if this operation is an piggyback candidate. */ - if (!new_ptr->is_dt) { + if (!op_ptr->is_dt) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ - new_ptr->piggyback_lock_candidate = 1; + op_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ - get_pkt = &(new_ptr->pkt.get); + get_pkt = &(op_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; @@ -392,7 +392,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -499,7 +499,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp } } else { - MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; MPI_Aint origin_type_size; size_t immed_len, len; @@ -511,7 +511,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp int i; /* queue it up */ - mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); + mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -519,25 +519,25 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp /******************** Setting operation struct areas ***********************/ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = origin_count; - new_ptr->origin_datatype = origin_datatype; - new_ptr->target_rank = target_rank; + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = origin_count; + op_ptr->origin_datatype = origin_datatype; + op_ptr->target_rank = target_rank; /* Remember user request */ if (ureq) { - new_ptr->ureq = ureq; + op_ptr->ureq = ureq; } /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, origin_dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, target_dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); @@ -577,7 +577,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPID_Datatype_is_contig(target_datatype, &is_target_contig); /* Judge if we can use IMMED data packet */ - if (!new_ptr->is_dt && is_origin_contig && is_target_contig) { + if (!op_ptr->is_dt && is_origin_contig && is_target_contig) { MPIU_Assign_trunc(immed_len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size, size_t); @@ -586,17 +586,17 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp } /* Judge if this operation is an piggyback candidate. */ - if (!new_ptr->is_dt) { + if (!op_ptr->is_dt) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) - new_ptr->piggyback_lock_candidate = 1; + op_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ - accum_pkt = &(new_ptr->pkt.accum); + accum_pkt = &(op_ptr->pkt.accum); if (use_immed_pkt) { MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE_IMMED); @@ -623,7 +623,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -733,10 +733,10 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, } } else { - MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_RMA_Op_t *op_ptr = NULL; /* Append the operation to the window's RMA ops queue */ - mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); + mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -754,25 +754,25 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /******************** Setting operation struct areas ***********************/ - new_ptr->origin_addr = result_addr; - new_ptr->origin_count = result_count; - new_ptr->origin_datatype = result_datatype; - new_ptr->target_rank = target_rank; + op_ptr->origin_addr = result_addr; + op_ptr->origin_count = result_count; + op_ptr->origin_datatype = result_datatype; + op_ptr->target_rank = target_rank; /* Remember user request */ if (ureq) { - new_ptr->ureq = ureq; + op_ptr->ureq = ureq; } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, dtp); MPID_Datatype_add_ref(dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } MPID_Datatype_get_size_macro(target_datatype, target_type_size); @@ -782,7 +782,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_is_contig(target_datatype, &is_target_contig); /* Judge if we can use IMMED data response packet */ - if (!new_ptr->is_dt && is_result_contig && is_target_contig) { + if (!op_ptr->is_dt && is_result_contig && is_target_contig) { MPIU_Assign_trunc(immed_len, (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size, size_t); @@ -791,16 +791,16 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, } /* Judge if this operation is a piggyback candidate */ - if (!new_ptr->is_dt) { + if (!op_ptr->is_dt) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ - new_ptr->piggyback_lock_candidate = 1; + op_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ - get_pkt = &(new_ptr->pkt.get); + get_pkt = &(op_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; @@ -826,32 +826,32 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, /******************** Setting operation struct areas ***********************/ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = origin_count; - new_ptr->origin_datatype = origin_datatype; - new_ptr->result_addr = result_addr; - new_ptr->result_count = result_count; - new_ptr->result_datatype = result_datatype; - new_ptr->target_rank = target_rank; + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = origin_count; + op_ptr->origin_datatype = origin_datatype; + op_ptr->result_addr = result_addr; + op_ptr->result_count = result_count; + op_ptr->result_datatype = result_datatype; + op_ptr->target_rank = target_rank; /* Remember user request */ if (ureq) { - new_ptr->ureq = ureq; + op_ptr->ureq = ureq; } /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, origin_dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, target_dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, result_dtp); - new_ptr->is_dt = 1; + op_ptr->is_dt = 1; } MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); @@ -895,7 +895,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPID_Datatype_is_contig(result_datatype, &is_result_contig); /* Judge if we can use IMMED data packet */ - if (!new_ptr->is_dt && is_origin_contig && is_target_contig && is_result_contig) { + if (!op_ptr->is_dt && is_origin_contig && is_target_contig && is_result_contig) { MPIU_Assign_trunc(immed_len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size, size_t); @@ -904,17 +904,17 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, } /* Judge if this operation is a piggyback candidate */ - if (!new_ptr->is_dt) { + if (!op_ptr->is_dt) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for origin, target and result data. We should extend this optimization to derived * datatypes as well. */ if (orig_len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) - new_ptr->piggyback_lock_candidate = 1; + op_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ - get_accum_pkt = &(new_ptr->pkt.get_accum); + get_accum_pkt = &(op_ptr->pkt.get_accum); if (use_immed_pkt) { MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED); @@ -941,7 +941,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -1149,13 +1149,13 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIU_ERR_POP(mpi_errno); } else { - MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL; MPI_Aint type_size; void *src = NULL, *dest = NULL; /* Append this operation to the RMA ops queue */ - mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); + mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1163,19 +1163,19 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, /******************** Setting operation struct areas ***********************/ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = 1; - new_ptr->origin_datatype = datatype; - new_ptr->result_addr = result_addr; - new_ptr->result_datatype = datatype; - new_ptr->compare_addr = (void *) compare_addr; - new_ptr->compare_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->piggyback_lock_candidate = 1; /* CAS is always able to piggyback LOCK */ + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = 1; + op_ptr->origin_datatype = datatype; + op_ptr->result_addr = result_addr; + op_ptr->result_datatype = datatype; + op_ptr->compare_addr = (void *) compare_addr; + op_ptr->compare_datatype = datatype; + op_ptr->target_rank = target_rank; + op_ptr->piggyback_lock_candidate = 1; /* CAS is always able to piggyback LOCK */ /************** Setting packet struct areas in operation ****************/ - cas_pkt = &(new_ptr->pkt.cas); + cas_pkt = &(op_ptr->pkt.cas); MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS_IMMED); cas_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; @@ -1200,7 +1200,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -1294,10 +1294,10 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIU_ERR_POP(mpi_errno); } else { - MPIDI_RMA_Op_t *new_ptr = NULL; + MPIDI_RMA_Op_t *op_ptr = NULL; /* Append this operation to the RMA ops queue */ - mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); + mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1313,11 +1313,11 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /******************** Setting operation struct areas ***********************/ - new_ptr->origin_addr = result_addr; - new_ptr->origin_count = 1; - new_ptr->origin_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->piggyback_lock_candidate = 1; + op_ptr->origin_addr = result_addr; + op_ptr->origin_count = 1; + op_ptr->origin_datatype = datatype; + op_ptr->target_rank = target_rank; + op_ptr->piggyback_lock_candidate = 1; /************** Setting packet struct areas in operation ****************/ @@ -1335,7 +1335,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, use_immed_resp_pkt = TRUE; } - get_pkt = &(new_ptr->pkt.get); + get_pkt = &(op_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; @@ -1356,13 +1356,13 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /******************** Setting operation struct areas ***********************/ - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = 1; - new_ptr->origin_datatype = datatype; - new_ptr->result_addr = result_addr; - new_ptr->result_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->piggyback_lock_candidate = 1; + op_ptr->origin_addr = (void *) origin_addr; + op_ptr->origin_count = 1; + op_ptr->origin_datatype = datatype; + op_ptr->result_addr = result_addr; + op_ptr->result_datatype = datatype; + op_ptr->target_rank = target_rank; + op_ptr->piggyback_lock_candidate = 1; /************** Setting packet struct areas in operation ****************/ @@ -1380,7 +1380,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, } } - fop_pkt = &(new_ptr->pkt.fop); + fop_pkt = &(op_ptr->pkt.fop); if (use_immed_pkt) { MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP_IMMED); @@ -1404,7 +1404,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); http://git.mpich.org/mpich.git/commitdiff/5a1dd95606ad83a8671a41ccc682076b9c... commit 5a1dd95606ad83a8671a41ccc682076b9c19d63b Author: Xin Zhao <[email protected]> Date: Mon Mar 9 11:32:25 2015 -0700 Delete macros that no longer needed. We already drop the code that uses those macros when re-implementing RMA, here we just delete those macros. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 6a23a9f..424ad1e 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -8,9 +8,6 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set); -#define MPIDI_PASSIVE_TARGET_DONE_TAG 348297 -#define MPIDI_PASSIVE_TARGET_RMA_TAG 563924 - /* === BEGIN_MPI_T_CVAR_INFO_BLOCK === http://git.mpich.org/mpich.git/commitdiff/1e321d463ba394c1b6fe9bc818c81be22e... commit 1e321d463ba394c1b6fe9bc818c81be22e39f24c Author: Xin Zhao <[email protected]> Date: Sun May 31 13:19:58 2015 -0500 Add a test to test atomic GET (GACC/FOP + MPI_NO_OP). Signed-off-by: Pavan Balaji <[email protected]> diff --git a/test/mpi/rma/Makefile.am b/test/mpi/rma/Makefile.am index f27e3be..656afc6 100644 --- a/test/mpi/rma/Makefile.am +++ b/test/mpi/rma/Makefile.am @@ -144,6 +144,7 @@ noinst_PROGRAMS = \ atomic_rmw_fop \ atomic_rmw_cas \ atomic_rmw_gacc \ + atomic_get \ acc-pairtype \ manyget \ derived-acc-flush_local\ diff --git a/test/mpi/rma/atomic_get.c b/test/mpi/rma/atomic_get.c new file mode 100644 index 0000000..54bffa6 --- /dev/null +++ b/test/mpi/rma/atomic_get.c @@ -0,0 +1,183 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2015 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +/* This test is going to test atomic GET (GACC/FOP+MPI_NO_OP). + * + * There are totally three processes involved in this test. Both + * rank 1 and rank 2 issue RMA operations to rank 0. Rank 2 issues + * atomic PUT (ACC+MPI_REPLACE), whereas rank 1 issues atomic + * GET (GACC/FOP+MPI_NO_OP). The datatype used in the test is + * pair-type. The initial value of pair-type data in origin buffer + * of atomic PUT is set so that the two basic values equal with + * each other. Due to the atomicity of GET, the expected resulting + * data should also have equivalent basic values. */ + +#include "mpi.h" +#include <stdio.h> + +#define LOOP 100 +#define DATA_SIZE 100 +#define OPS_NUM 10000 +#define GACC_SZ 10 + +typedef struct pair_struct { + long double a; + int b; +} pair_struct_t; + +int main(int argc, char *argv[]) +{ + int rank, nproc; + int i, j, k; + int errors = 0, curr_errors = 0; + MPI_Win win; + pair_struct_t *tar_buf = NULL; + pair_struct_t *orig_buf = NULL; + pair_struct_t *result_buf = NULL; + + /* This test needs to work with 3 processes. */ + + MPI_Init(&argc, &argv); + + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + MPI_Alloc_mem(sizeof(pair_struct_t) * DATA_SIZE, MPI_INFO_NULL, &orig_buf); + MPI_Alloc_mem(sizeof(pair_struct_t) * DATA_SIZE, MPI_INFO_NULL, &result_buf); + + MPI_Win_allocate(sizeof(pair_struct_t) * DATA_SIZE, sizeof(pair_struct_t), + MPI_INFO_NULL, MPI_COMM_WORLD, &tar_buf, &win); + + for (j = 0; j < LOOP * 6; j++) { + + MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win); + + /* initialize data */ + for (i = 0; i < DATA_SIZE; i++) { + tar_buf[i].a = 0.0; + tar_buf[i].b = 0; + result_buf[i].a = 0.0; + result_buf[i].b = 0; + } + + MPI_Win_unlock(rank, win); + + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_fence(0, win); + + if (rank == 2) { + if (j < 2 * LOOP) { + /* Work with FOP test (Test #1 to Test #2) */ + for (i = 0; i < OPS_NUM; i++) { + + int curr_val = j * OPS_NUM + i; + orig_buf[0].a = (long double) (curr_val); + orig_buf[0].b = curr_val; + + MPI_Accumulate(orig_buf, 1, MPI_LONG_DOUBLE_INT, + 0, 0, 1, MPI_LONG_DOUBLE_INT, MPI_REPLACE, win); + } + } + else { + /* Work with GACC test (Test #3 to Test #6) */ + for (i = 0; i < OPS_NUM / GACC_SZ; i++) { + + for (k = 0; k < GACC_SZ; k++) { + int curr_val = j * OPS_NUM + i * GACC_SZ + k; + orig_buf[k].a = (long double) (curr_val); + orig_buf[k].b = curr_val; + } + + MPI_Accumulate(orig_buf, GACC_SZ, MPI_LONG_DOUBLE_INT, + 0, 0, GACC_SZ, MPI_LONG_DOUBLE_INT, MPI_REPLACE, win); + } + } + } + else if (rank == 1) { + /* equals to an atomic GET */ + if (j < LOOP) { + for (i = 0; i < DATA_SIZE; i++) { + /* Test #1: FOP + MPI_NO_OP */ + MPI_Fetch_and_op(orig_buf, &(result_buf[i]), MPI_LONG_DOUBLE_INT, + 0, 0, MPI_NO_OP, win); + } + } + else if (j < 2 * LOOP) { + for (i = 0; i < DATA_SIZE; i++) { + /* Test #2: FOP + MPI_NO_OP + NULL origin buffer address */ + MPI_Fetch_and_op(NULL, &(result_buf[i]), MPI_LONG_DOUBLE_INT, + 0, 0, MPI_NO_OP, win); + } + } + else if (j < 3 * LOOP) { + for (i = 0; i < DATA_SIZE / GACC_SZ; i++) { + /* Test #3: GACC + MPI_NO_OP */ + MPI_Get_accumulate(orig_buf, GACC_SZ, MPI_LONG_DOUBLE_INT, + &(result_buf[i * GACC_SZ]), GACC_SZ, MPI_LONG_DOUBLE_INT, + 0, 0, GACC_SZ, MPI_LONG_DOUBLE_INT, MPI_NO_OP, win); + } + } + else if (j < 4 * LOOP) { + for (i = 0; i < DATA_SIZE / GACC_SZ; i++) { + /* Test #4: GACC + MPI_NO_OP + NULL origin buffer address */ + MPI_Get_accumulate(NULL, GACC_SZ, MPI_LONG_DOUBLE_INT, + &(result_buf[i * GACC_SZ]), GACC_SZ, MPI_LONG_DOUBLE_INT, + 0, 0, GACC_SZ, MPI_LONG_DOUBLE_INT, MPI_NO_OP, win); + } + } + else if (j < 5 * LOOP) { + for (i = 0; i < DATA_SIZE / GACC_SZ; i++) { + /* Test #5: GACC + MPI_NO_OP + zero origin count */ + MPI_Get_accumulate(orig_buf, 0, MPI_LONG_DOUBLE_INT, + &(result_buf[i * GACC_SZ]), GACC_SZ, MPI_LONG_DOUBLE_INT, + 0, 0, GACC_SZ, MPI_LONG_DOUBLE_INT, MPI_NO_OP, win); + } + } + else if (j < 6 * LOOP) { + for (i = 0; i < DATA_SIZE / GACC_SZ; i++) { + /* Test #5: GACC + MPI_NO_OP + NULL origin datatype */ + MPI_Get_accumulate(orig_buf, GACC_SZ, MPI_DATATYPE_NULL, + &(result_buf[i * GACC_SZ]), GACC_SZ, MPI_LONG_DOUBLE_INT, + 0, 0, GACC_SZ, MPI_LONG_DOUBLE_INT, MPI_NO_OP, win); + } + } + } + + MPI_Win_fence(0, win); + + /* check results */ + if (rank == 1) { + for (i = 0; i < DATA_SIZE; i++) { + if (result_buf[i].a != (long double) (result_buf[i].b)) { + if (curr_errors < 10) { + printf("LOOP %d: result_buf[%d].a = %Lf, result_buf[%d].b = %d\n", + j, i, result_buf[i].a, i, result_buf[i].b); + } + curr_errors++; + } + } + } + + if (j % LOOP == 0) { + errors += curr_errors; + curr_errors = 0; + } + } + + MPI_Win_free(&win); + + MPI_Free_mem(orig_buf); + MPI_Free_mem(result_buf); + + if (rank == 1) { + if (errors == 0) + printf(" No Errors\n"); + } + + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/rma/testlist.in b/test/mpi/rma/testlist.in index e0f994b..475cc43 100644 --- a/test/mpi/rma/testlist.in +++ b/test/mpi/rma/testlist.in @@ -131,6 +131,7 @@ at_complete 2 atomic_rmw_fop 3 atomic_rmw_cas 3 atomic_rmw_gacc 3 +atomic_get 3 mpiversion=3.0 aint 2 mpiversion=3.1 acc-pairtype 2 manyget 2 ----------------------------------------------------------------------- Summary of changes: src/mpi/rma/fetch_and_op.c | 4 +- src/mpi/rma/get_accumulate.c | 4 + .../ch3/channels/nemesis/include/mpid_nem_inline.h | 3 +- .../ch3/channels/nemesis/include/mpidi_ch3_pre.h | 2 +- src/mpid/ch3/channels/nemesis/src/ch3_progress.c | 4 +- src/mpid/ch3/channels/sock/include/mpidi_ch3_pre.h | 2 +- src/mpid/ch3/channels/sock/src/ch3_progress.c | 6 +- src/mpid/ch3/include/mpid_rma_issue.h | 106 +++- src/mpid/ch3/include/mpid_rma_oplist.h | 257 ++++----- src/mpid/ch3/include/mpid_rma_shm.h | 65 ++- src/mpid/ch3/include/mpid_rma_types.h | 20 +- src/mpid/ch3/include/mpidpkt.h | 23 + src/mpid/ch3/include/mpidrma.h | 130 +++-- src/mpid/ch3/src/ch3u_handle_recv_req.c | 154 +++-- src/mpid/ch3/src/ch3u_rma_ops.c | 610 ++++++++------------ src/mpid/ch3/src/ch3u_rma_pkthandler.c | 111 +++-- src/mpid/ch3/src/ch3u_rma_progress.c | 349 ++++++------ src/mpid/ch3/src/ch3u_rma_sync.c | 105 ++-- src/mpid/ch3/src/mpid_rma.c | 2 +- test/mpi/rma/Makefile.am | 1 + test/mpi/rma/atomic_get.c | 183 ++++++ test/mpi/rma/testlist.in | 1 + 22 files changed, 1141 insertions(+), 1001 deletions(-) create mode 100644 test/mpi/rma/atomic_get.c hooks/post-receive -- MPICH primary repository