[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.2a2-162-g3df5868

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3df5868917cae81c72f098100f80ff2f79985905 (commit) via 21126e9e6a565575078b89e7cb563bb2aec2c5a9 (commit) via e3ccad1f165fc730aad3a6bd5b6338a9c42b1087 (commit) via e3fc7e702299543e7d6e9d07a169ef71517a7208 (commit) via 80a71e113f179cd13eef8f0e9c7481332df06d72 (commit) via 42c52518e0179196e8dd8021f6e72cbaa9a417fd (commit) via c8ecef8d531601900e9d8db58284897711295d12 (commit) via 59afc29cbf253c7cb7f8efb052e816338cb99304 (commit) via 1b30ab19a3dfdd6ae31c87dba1928ee68ec2775a (commit) via 21479b00b1080d577be8ff7ed66a7bdd62dd16ac (commit) via de9d0f21b33691ee555d3479acdc5e5863af92ca (commit) via 3a017faae893f03d2ad2fb4d24f037d964782e23 (commit) via 81e2b2742b9a690e68d40d93e42a6d3684206fa3 (commit) via d46b848a7e574a92521abe02bb63ba9275512d14 (commit) via 064e60ce4e6903c56c27db5d6d8790f4be05f491 (commit) via fa95883370a01f71aeabeee548e4076880ecbd86 (commit) via 2cbc9180045dea815a7aeb7a7d1fe7f299020f72 (commit) via bb3f96236aaab668a2129ec4939976588fce4ebf (commit) from b21dc461115bdf99d42176ecea8e8efc763d8a2d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3df5868917cae81c72f098100f80ff2f79... commit 3df5868917cae81c72f098100f80ff2f79985905 Author: Xin Zhao <[email protected]> Date: Wed Feb 11 02:15:17 2015 -0600 Bug-fix: add barriers between init buffers and issuing RMA operations. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/test/mpi/rma/atomic_rmw_gacc.c b/test/mpi/rma/atomic_rmw_gacc.c index d04e9c3..8bd7582 100644 --- a/test/mpi/rma/atomic_rmw_gacc.c +++ b/test/mpi/rma/atomic_rmw_gacc.c @@ -149,7 +149,9 @@ int main (int argc, char *argv[]) { MPI_Win_unlock(rank, win); } - MPI_Win_lock_all(0, win); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); if (rank != dest) { for (i = 0; i < my_buf_num; i++) { MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), 1, origin_dtp, @@ -176,7 +178,9 @@ int main (int argc, char *argv[]) { MPI_Win_unlock(rank, win); } - MPI_Win_lock_all(0, win); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); if (rank != dest) { for (i = 0; i < my_buf_num; i++) { MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), OP_COUNT, MPI_INT, @@ -203,7 +207,9 @@ int main (int argc, char *argv[]) { MPI_Win_unlock(rank, win); } - MPI_Win_lock_all(0, win); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); if (rank != dest) { for (i = 0; i < my_buf_num; i++) { MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), 1, origin_dtp, http://git.mpich.org/mpich.git/commitdiff/21126e9e6a565575078b89e7cb563bb2ae... commit 21126e9e6a565575078b89e7cb563bb2aec2c5a9 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 13:51:03 2015 -0800 Delete comments that no longer make sense. The comments are no longer significant for new RMA infrastructure. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 699a1f3..aa2db5b 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -591,13 +591,8 @@ typedef struct MPIDI_CH3_Pkt_put { void *addr; int count; MPI_Datatype datatype; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ + MPI_Win target_win_handle; + MPI_Win source_win_handle; union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -617,10 +612,7 @@ typedef struct MPIDI_CH3_Pkt_get { int dataloop_size; /* for derived datatypes */ } info; MPI_Request request_handle; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; } MPIDI_CH3_Pkt_get_t; typedef struct MPIDI_CH3_Pkt_get_resp { @@ -645,13 +637,8 @@ typedef struct MPIDI_CH3_Pkt_accum { int count; MPI_Datatype datatype; MPI_Op op; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ + MPI_Win target_win_handle; + MPI_Win source_win_handle; union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -666,10 +653,7 @@ typedef struct MPIDI_CH3_Pkt_get_accum { int count; MPI_Datatype datatype; MPI_Op op; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -697,10 +681,7 @@ typedef struct MPIDI_CH3_Pkt_cas { MPI_Datatype datatype; void *addr; MPI_Request request_handle; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; MPIDI_CH3_CAS_Immed_u origin_data; MPIDI_CH3_CAS_Immed_u compare_data; } MPIDI_CH3_Pkt_cas_t; @@ -726,10 +707,7 @@ typedef struct MPIDI_CH3_Pkt_fop { void *addr; MPI_Op op; MPI_Request request_handle; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; struct { /* note that we use struct here in order to consistently access data @@ -786,8 +764,7 @@ typedef struct MPIDI_CH3_Pkt_lock_ack { or request_handle is used. */ MPI_Win source_win_handle; MPI_Request request_handle; - int target_rank; /* Used in flush_ack response to look up the - * target state at the origin. */ + int target_rank; } MPIDI_CH3_Pkt_lock_ack_t; typedef struct MPIDI_CH3_Pkt_lock_op_ack { @@ -803,8 +780,7 @@ typedef struct MPIDI_CH3_Pkt_lock_op_ack { typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; MPI_Win source_win_handle; - int target_rank; /* Used in flush_ack response to look up the - * target state at the origin. */ + int target_rank; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_flush_ack_t; http://git.mpich.org/mpich.git/commitdiff/e3ccad1f165fc730aad3a6bd5b6338a9c4... commit e3ccad1f165fc730aad3a6bd5b6338a9c42b1087 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 12:00:25 2015 -0800 Delete unnecessary code. Here req->dev.user_count is used when receiving FOP/CAS response data on origin in PktHandler_FOPResp and PktHandler_CASResp. Since the count always be 1, we did not set rma_op->result_count, and we directly set req->dev.user_count to 1 in packet handlers. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index eda0085..47b2569 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -667,7 +667,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, MPIU_Object_set_ref(rma_op->request, 2); rma_op->request->dev.user_buf = rma_op->result_addr; - rma_op->request->dev.user_count = rma_op->result_count; rma_op->request->dev.datatype = rma_op->result_datatype; rma_op->request->dev.target_win_handle = MPI_WIN_NULL; @@ -733,7 +732,6 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, MPIU_Object_set_ref(resp_req, 2); resp_req->dev.user_buf = rma_op->result_addr; - resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; resp_req->dev.target_win_handle = MPI_WIN_NULL; resp_req->dev.source_win_handle = win_ptr->handle; http://git.mpich.org/mpich.git/commitdiff/e3fc7e702299543e7d6e9d07a169ef7151... commit e3fc7e702299543e7d6e9d07a169ef71517a7208 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 11:52:19 2015 -0800 Simplify code of issuing RMA packets. When issuing RMA packets, we do not need to store target_win_handle in the request on origin side but only need to store source_win_handle. Because when the response data is back, we only needs to use source_win_handle on origin size. This patch simplifies the code in this way. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 7cb4052..eda0085 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -457,7 +457,7 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, resp_req->dev.user_buf = rma_op->result_addr; resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; + resp_req->dev.target_win_handle = MPI_WIN_NULL; resp_req->dev.source_win_handle = win_ptr->handle; if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { @@ -670,7 +670,7 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, rma_op->request->dev.user_count = rma_op->result_count; rma_op->request->dev.datatype = rma_op->result_datatype; - rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; + rma_op->request->dev.target_win_handle = MPI_WIN_NULL; rma_op->request->dev.source_win_handle = win_ptr->handle; cas_pkt->request_handle = rma_op->request->handle; @@ -735,7 +735,7 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, resp_req->dev.user_buf = rma_op->result_addr; resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = fop_pkt->target_win_handle; + resp_req->dev.target_win_handle = MPI_WIN_NULL; resp_req->dev.source_win_handle = win_ptr->handle; fop_pkt->request_handle = resp_req->handle; http://git.mpich.org/mpich.git/commitdiff/80a71e113f179cd13eef8f0e9c7481332d... commit 80a71e113f179cd13eef8f0e9c7481332df06d72 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 11:37:50 2015 -0800 Remove source_win_handle from GET-like RMA packets. For GET-like RMA packets and response packets (GACC, GET, FOP, CAS, GACC_RESP, GET_RESP, FOP_RESP, CAS_RESP), originally we carry source_win_handle in packet struct in order to locate window handle on origin side in the packet handler of response packets. However, this is not necessary because source_win_handle can be stored in the request on the origin side. This patch delete source_win_handle from those packets to reduce the size of packet union. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 2c1a821..7cb4052 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -458,7 +458,7 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; - resp_req->dev.source_win_handle = get_accum_pkt->source_win_handle; + resp_req->dev.source_win_handle = win_ptr->handle; if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { MPID_Datatype *result_dtp = NULL; @@ -572,7 +572,7 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, rma_op->request->dev.user_count = rma_op->origin_count; rma_op->request->dev.datatype = rma_op->origin_datatype; rma_op->request->dev.target_win_handle = MPI_WIN_NULL; - rma_op->request->dev.source_win_handle = get_pkt->source_win_handle; + rma_op->request->dev.source_win_handle = win_ptr->handle; if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->request->dev.datatype)) { MPID_Datatype_get_ptr(rma_op->request->dev.datatype, dtp); rma_op->request->dev.datatype_ptr = dtp; @@ -671,7 +671,7 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, rma_op->request->dev.datatype = rma_op->result_datatype; rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; - rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle; + rma_op->request->dev.source_win_handle = win_ptr->handle; cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags |= flags; @@ -736,7 +736,7 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; resp_req->dev.target_win_handle = fop_pkt->target_win_handle; - resp_req->dev.source_win_handle = fop_pkt->source_win_handle; + resp_req->dev.source_win_handle = win_ptr->handle; fop_pkt->request_handle = resp_req->handle; diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index cbd300c..699a1f3 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -442,39 +442,10 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_PUT_IMMED): \ win_hdl_ = (pkt_).put.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_GET): \ - win_hdl_ = (pkt_).get.source_win_handle; \ - break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ win_hdl_ = (pkt_).accum.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ - win_hdl_ = (pkt_).get_accum.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_CAS_IMMED): \ - win_hdl_ = (pkt_).cas.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - case (MPIDI_CH3_PKT_FOP_IMMED): \ - win_hdl_ = (pkt_).fop.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_GET_RESP): \ - case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ - win_hdl_ = (pkt_).get_resp.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ - win_hdl_ = (pkt_).get_accum_resp.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_FOP_RESP): \ - case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ - win_hdl_ = (pkt_).fop_resp.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ - win_hdl_ = (pkt_).cas_resp.source_win_handle; \ - break; \ case (MPIDI_CH3_PKT_LOCK): \ win_hdl_ = (pkt_).lock.source_win_handle; \ break; \ @@ -567,6 +538,53 @@ MPIDI_CH3_PKT_DEFS } \ } +#define MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE(pkt_, request_hdl_, err_) \ + { \ + err_ = MPI_SUCCESS; \ + switch((pkt_).type) { \ + case (MPIDI_CH3_PKT_GET): \ + request_hdl_ = (pkt_).get.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ + request_hdl_ = (pkt_).get_accum.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ + request_hdl_ = (pkt_).cas.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ + request_hdl_ = (pkt_).fop.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ + request_hdl_ = (pkt_).get_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ + request_hdl_ = (pkt_).get_accum_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ + request_hdl_ = (pkt_).fop_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ + request_hdl_ = (pkt_).cas_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK): \ + request_hdl_ = (pkt_).lock.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + request_hdl_ = (pkt_).lock_ack.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + request_hdl_ = (pkt_).lock_op_ack.request_handle; \ + break; \ + default: \ + MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ + } \ + } + typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; @@ -603,9 +621,6 @@ typedef struct MPIDI_CH3_Pkt_get { * epoch for decrementing rma op counter in * active target rma and for unlocking window * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ } MPIDI_CH3_Pkt_get_t; typedef struct MPIDI_CH3_Pkt_get_resp { @@ -613,7 +628,6 @@ typedef struct MPIDI_CH3_Pkt_get_resp { MPI_Request request_handle; /* followings are used to decrement ack_counter at origin */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ struct { @@ -656,9 +670,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum { * epoch for decrementing rma op counter in * active target rma and for unlocking window * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -670,7 +681,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { MPI_Request request_handle; /* followings are used to decrement ack_counter at origin */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ struct { @@ -687,7 +697,6 @@ typedef struct MPIDI_CH3_Pkt_cas { MPI_Datatype datatype; void *addr; MPI_Request request_handle; - MPI_Win source_win_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -707,7 +716,6 @@ typedef struct MPIDI_CH3_Pkt_cas_resp { } info; /* followings are used to decrement ack_counter at orign */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_cas_resp_t; @@ -718,7 +726,6 @@ typedef struct MPIDI_CH3_Pkt_fop { void *addr; MPI_Op op; MPI_Request request_handle; - MPI_Win source_win_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -742,7 +749,6 @@ typedef struct MPIDI_CH3_Pkt_fop_resp { } info; /* followings are used to decrement ack_counter at orign */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_fop_resp_t; @@ -750,7 +756,14 @@ typedef struct MPIDI_CH3_Pkt_lock { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; + /* Note that either source_win_handle + or request_handle will be used. Here + we need both of them because PUT/GET + may be converted to LOCK packet, + PUT has source_win_handle area and + GET has request_handle area. */ MPI_Win source_win_handle; + MPI_Request request_handle; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { @@ -769,7 +782,10 @@ typedef struct MPIDI_CH3_Pkt_flush { typedef struct MPIDI_CH3_Pkt_lock_ack { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; + /* note that either source_win_handle + or request_handle is used. */ MPI_Win source_win_handle; + MPI_Request request_handle; int target_rank; /* Used in flush_ack response to look up the * target state at the origin. */ } MPIDI_CH3_Pkt_lock_ack_t; @@ -777,7 +793,10 @@ typedef struct MPIDI_CH3_Pkt_lock_ack { typedef struct MPIDI_CH3_Pkt_lock_op_ack { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; + /* note that either source_win_handle + or request_handle is used. */ MPI_Win source_win_handle; + MPI_Request request_handle; int target_rank; } MPIDI_CH3_Pkt_lock_op_ack_t; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 1b4fdf9..c60124a 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -32,6 +32,7 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; lock_pkt->source_win_handle = win_ptr->handle; + lock_pkt->request_handle = MPI_REQUEST_NULL; lock_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (lock_type == MPI_LOCK_SHARED) lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; @@ -110,7 +111,8 @@ static inline int send_unlock_msg(int dest, MPID_Win * win_ptr, #define FCNAME MPIDI_QUOTE(FUNCNAME) static inline int MPIDI_CH3I_Send_lock_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle) + MPI_Win source_win_handle, + MPI_Request request_handle) { MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_ack_t *lock_ack_pkt = &upkt.lock_ack; @@ -120,9 +122,12 @@ static inline int MPIDI_CH3I_Send_lock_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_p MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_ACK_PKT); + MPIU_Assert(!(source_win_handle != MPI_WIN_NULL && request_handle != MPI_REQUEST_NULL)); + /* send lock ack packet */ MPIDI_Pkt_init(lock_ack_pkt, MPIDI_CH3_PKT_LOCK_ACK); lock_ack_pkt->source_win_handle = source_win_handle; + lock_ack_pkt->request_handle = request_handle; lock_ack_pkt->target_rank = win_ptr->comm_ptr->rank; lock_ack_pkt->flags = flags; @@ -153,7 +158,8 @@ static inline int MPIDI_CH3I_Send_lock_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_p #define FCNAME MPIDI_QUOTE(FUNCNAME) static inline int MPIDI_CH3I_Send_lock_op_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle) + MPI_Win source_win_handle, + MPI_Request request_handle) { MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_op_ack_t *lock_op_ack_pkt = &upkt.lock_op_ack; @@ -163,9 +169,12 @@ static inline int MPIDI_CH3I_Send_lock_op_ack_pkt(MPIDI_VC_t * vc, MPID_Win * wi MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_OP_ACK_PKT); + MPIU_Assert(!(source_win_handle != MPI_WIN_NULL && request_handle != MPI_REQUEST_NULL)); + /* send lock ack packet */ MPIDI_Pkt_init(lock_op_ack_pkt, MPIDI_CH3_PKT_LOCK_OP_ACK); lock_op_ack_pkt->source_win_handle = source_win_handle; + lock_op_ack_pkt->request_handle = request_handle; lock_op_ack_pkt->target_rank = win_ptr->comm_ptr->rank; lock_op_ack_pkt->flags = flags; @@ -318,6 +327,7 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_RMA_Lock_entry_t *new_ptr = NULL; MPIDI_CH3_Pkt_flags_t flag; MPI_Win source_win_handle; + MPI_Request request_handle; int lock_discarded = 0, data_discarded = 0; int mpi_errno = MPI_SUCCESS; @@ -385,12 +395,21 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Pkt_flags_t flags; MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE((*pkt), target_win_handle, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); MPIDI_CH3_PKT_RMA_GET_FLAGS((*pkt), flags, mpi_errno); + if (pkt->type == MPIDI_CH3_PKT_PUT || pkt->type == MPIDI_CH3_PKT_ACCUMULATE) { + MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); + request_handle = MPI_REQUEST_NULL; + } + else { + source_win_handle = MPI_WIN_NULL; + MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE((*pkt), request_handle, mpi_errno); + } + MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = target_win_handle; lock_pkt->source_win_handle = source_win_handle; + lock_pkt->request_handle = request_handle; lock_pkt->flags = flags; /* replace original pkt with lock pkt */ @@ -456,12 +475,14 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, } issue_ack: - MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); if (pkt->type == MPIDI_CH3_PKT_LOCK) { if (lock_discarded) flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED; else flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED; - mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, flag, source_win_handle); + MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); + MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE((*pkt), request_handle, mpi_errno); + + mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, flag, source_win_handle, request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { @@ -469,7 +490,17 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, else if (data_discarded) flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED; else flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED; - mpi_errno = MPIDI_CH3I_Send_lock_op_ack_pkt(vc, win_ptr, flag, source_win_handle); + if (pkt->type == MPIDI_CH3_PKT_PUT || pkt->type == MPIDI_CH3_PKT_PUT_IMMED || + pkt->type == MPIDI_CH3_PKT_ACCUMULATE || pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { + MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); + request_handle = MPI_REQUEST_NULL; + } + else { + source_win_handle = MPI_WIN_NULL; + MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE((*pkt), request_handle, mpi_errno); + } + + mpi_errno = MPIDI_CH3I_Send_lock_op_ack_pkt(vc, win_ptr, flag, source_win_handle, request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -864,9 +895,11 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, if ((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) pkt_flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; + MPIU_Assert(source_win_handle != MPI_WIN_NULL); mpi_errno = MPIDI_CH3I_Send_lock_op_ack_pkt(vc, win_ptr, pkt_flags, - source_win_handle); + source_win_handle, + MPI_REQUEST_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); MPIDI_CH3_Progress_signal_completion(); } diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 2f3f9ef..d1ae95d 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -240,7 +240,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, } get_accum_resp_pkt->request_handle = rreq->dev.resp_request_handle; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_accum_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) @@ -418,7 +417,6 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *vc, /* Send back data */ MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); fop_resp_pkt->request_handle = rreq->dev.resp_request_handle; - fop_resp_pkt->source_win_handle = rreq->dev.source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -694,13 +692,11 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, sreq->dev.datatype = new_dtp->handle; sreq->dev.datatype_ptr = new_dtp; sreq->dev.target_win_handle = rreq->dev.target_win_handle; - sreq->dev.source_win_handle = rreq->dev.source_win_handle; sreq->dev.flags = rreq->dev.flags; MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); get_resp_pkt->request_handle = rreq->dev.request_handle; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) @@ -1022,7 +1018,6 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_resp_pkt->source_win_handle = get_pkt->source_win_handle; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); @@ -1211,7 +1206,6 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* All origin data is in packet header, issue the header. */ @@ -1270,7 +1264,6 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } fop_resp_pkt->request_handle = fop_pkt->request_handle; - fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1379,7 +1372,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* do final action */ mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, - fop_pkt->flags, fop_pkt->source_win_handle); + fop_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); fn_exit: @@ -1406,7 +1399,6 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; - cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1461,7 +1453,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* do final action */ mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, - cas_pkt->flags, cas_pkt->source_win_handle); + cas_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); fn_exit: @@ -1492,7 +1484,8 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent else { mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(lock_entry->vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, - lock_pkt->source_win_handle); + lock_pkt->source_win_handle, + lock_pkt->request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } } diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c index 93d8e5d..6d0f81d 100644 --- a/src/mpid/ch3/src/ch3u_handle_send_req.c +++ b/src/mpid/ch3/src/ch3u_handle_send_req.c @@ -49,7 +49,6 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = sreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = sreq->dev.flags; /* NOTE: It is possible that this request is already completed before @@ -85,7 +84,7 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; @@ -106,7 +105,6 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc, { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = rreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = rreq->dev.flags; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMSENDCOMPLETE); @@ -150,7 +148,7 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc, on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; @@ -174,7 +172,6 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = rreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = rreq->dev.flags; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_CASSENDCOMPLETE); @@ -218,7 +215,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE/* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; @@ -240,7 +237,6 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc, { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = rreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = rreq->dev.flags; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPSENDCOMPLETE); @@ -284,7 +280,7 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc, on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index a43a606..5497210 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -373,7 +373,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype get_pkt->datatype = target_datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; @@ -747,7 +746,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_pkt->datatype = target_datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt == TRUE) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; @@ -838,7 +836,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_accum_pkt->info.dataloop_size = 0; get_accum_pkt->op = op; get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_accum_pkt->source_win_handle = win_ptr->handle; get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { void *src = (void *)origin_addr, *dest = (void *)(get_accum_pkt->info.data); @@ -1090,7 +1087,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, win_ptr->disp_units[target_rank] * target_disp; cas_pkt->datatype = datatype; cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - cas_pkt->source_win_handle = win_ptr->handle; cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; /* REQUIRE: All datatype arguments must be of the same, builtin @@ -1245,7 +1241,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, get_pkt->datatype = datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt == TRUE) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; @@ -1293,7 +1288,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, win_ptr->disp_units[target_rank] * target_disp; fop_pkt->datatype = datatype; fop_pkt->op = op; - fop_pkt->source_win_handle = win_ptr->handle; fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 0f4674e..ab77790 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -382,7 +382,6 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req = MPID_Request_create(); req->dev.target_win_handle = get_pkt->target_win_handle; - req->dev.source_win_handle = get_pkt->source_win_handle; req->dev.flags = get_pkt->flags; /* get start location of data and length of data */ @@ -424,7 +423,6 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_resp_pkt->source_win_handle = get_pkt->source_win_handle; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); @@ -764,7 +762,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, resp_req = MPID_Request_create(); resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; - resp_req->dev.source_win_handle = get_accum_pkt->source_win_handle; resp_req->dev.flags = get_accum_pkt->flags; MPIDI_Request_set_type(resp_req, MPIDI_REQUEST_TYPE_GET_ACCUM_RESP); @@ -796,7 +793,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) @@ -868,7 +864,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.op = get_accum_pkt->op; req->dev.real_user_buf = get_accum_pkt->addr; req->dev.target_win_handle = get_accum_pkt->target_win_handle; - req->dev.source_win_handle = get_accum_pkt->source_win_handle; req->dev.flags = get_accum_pkt->flags; req->dev.resp_request_handle = get_accum_pkt->request_handle; @@ -1024,7 +1019,6 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; - cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1078,7 +1072,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - cas_pkt->flags, cas_pkt->source_win_handle); + cas_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); fn_exit: @@ -1113,7 +1107,8 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_cas_resp); - MPID_Win_get_ptr(cas_resp_pkt->source_win_handle, win_ptr); + MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter on this target */ if (cas_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1130,7 +1125,6 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); MPID_Datatype_get_size_macro(req->dev.datatype, len); MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->info.data, len); @@ -1196,7 +1190,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED); fop_resp_pkt->request_handle = fop_pkt->request_handle; - fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1256,7 +1249,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - fop_pkt->flags, fop_pkt->source_win_handle); + fop_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { @@ -1277,7 +1270,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.op = fop_pkt->op; req->dev.real_user_buf = fop_pkt->addr; req->dev.target_win_handle = fop_pkt->target_win_handle; - req->dev.source_win_handle = fop_pkt->source_win_handle; req->dev.flags = fop_pkt->flags; req->dev.resp_request_handle = fop_pkt->request_handle; req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPRecvComplete; @@ -1354,9 +1346,8 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_fop_resp); - MPID_Win_get_ptr(fop_resp_pkt->source_win_handle, win_ptr); - MPID_Request_get_ptr(fop_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter */ if (fop_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1440,7 +1431,8 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_accum_resp); - MPID_Win_get_ptr(get_accum_resp_pkt->source_win_handle, win_ptr); + MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter on target */ if (get_accum_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1460,8 +1452,6 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req); - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; @@ -1538,7 +1528,8 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 1) { /* send lock granted packet. */ mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, - lock_pkt->source_win_handle); + lock_pkt->source_win_handle, + lock_pkt->request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -1582,7 +1573,8 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_resp); - MPID_Win_get_ptr(get_resp_pkt->source_win_handle, win_ptr); + MPID_Request_get_ptr(get_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter on target */ if (get_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1602,8 +1594,6 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - MPID_Request_get_ptr(get_resp_pkt->request_handle, req); - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; @@ -1670,7 +1660,17 @@ int MPIDI_CH3_PktHandler_LockAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); + if (lock_ack_pkt->source_win_handle != MPI_WIN_NULL) { MPID_Win_get_ptr(lock_ack_pkt->source_win_handle, win_ptr); + } + else { + MPIU_Assert(lock_ack_pkt->request_handle != MPI_REQUEST_NULL); + + MPID_Request *req_ptr = NULL; + MPID_Request_get_ptr(lock_ack_pkt->request_handle, req_ptr); + MPIU_Assert(req_ptr->dev.source_win_handle != MPI_REQUEST_NULL); + MPID_Win_get_ptr(req_ptr->dev.source_win_handle, win_ptr); + } mpi_errno = handle_lock_ack(win_ptr, target_rank, lock_ack_pkt->flags); @@ -1705,7 +1705,17 @@ int MPIDI_CH3_PktHandler_LockOpAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); + if (lock_op_ack_pkt->source_win_handle != MPI_WIN_NULL) { MPID_Win_get_ptr(lock_op_ack_pkt->source_win_handle, win_ptr); + } + else { + MPIU_Assert(lock_op_ack_pkt->request_handle != MPI_REQUEST_NULL); + + MPID_Request *req_ptr = NULL; + MPID_Request_get_ptr(lock_op_ack_pkt->request_handle, req_ptr); + MPIU_Assert(req_ptr->dev.source_win_handle != MPI_REQUEST_NULL); + MPID_Win_get_ptr(req_ptr->dev.source_win_handle, win_ptr); + } mpi_errno = handle_lock_ack(win_ptr, target_rank, lock_op_ack_pkt->flags); http://git.mpich.org/mpich.git/commitdiff/42c52518e0179196e8dd8021f6e72cbaa9... commit 42c52518e0179196e8dd8021f6e72cbaa9a417fd Author: Xin Zhao <[email protected]> Date: Tue Feb 10 10:26:45 2015 -0800 Bug-fix: when encounter mpich failure, unlock inter-process lock. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 5943d35..2f3f9ef 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -268,6 +268,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, void *src = (void *)(rreq->dev.real_user_buf), *dest = (void *)(get_accum_resp_pkt->info.data); mpi_errno = immed_copy(src, dest, rreq->dev.user_count * type_size); if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); MPIU_ERR_POP(mpi_errno); } } @@ -279,6 +281,10 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Segment *seg = MPID_Segment_alloc(); MPI_Aint last = type_size * rreq->dev.user_count; + if (seg == NULL) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + } MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment"); MPID_Segment_init(rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, seg, 0); MPID_Segment_pack(seg, 0, &last, resp_req->dev.user_buf); @@ -289,13 +295,12 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, /* accumulate data from tmp_buf into user_buf */ mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumSendComplete; resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumSendComplete; resp_req->dev.target_win_handle = rreq->dev.target_win_handle; @@ -403,14 +408,13 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *vc, if (rreq->dev.op != MPI_NO_OP) { mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, 1, rreq->dev.datatype, rreq->dev.op); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + /* Send back data */ MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); fop_resp_pkt->request_handle = rreq->dev.resp_request_handle; @@ -1074,19 +1078,19 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* All data fits in packet header */ mpi_errno = do_accumulate_op(acc_pkt->info.data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { MPIU_Assert(acc_pkt->type == MPIDI_CH3_PKT_ACCUMULATE); mpi_errno = do_accumulate_op(lock_entry->data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, FALSE /* has no response data */, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1148,7 +1152,11 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { void *src = (void *)(get_accum_pkt->addr), *dest = (void *)(get_accum_resp_pkt->info.data); mpi_errno = immed_copy(src, dest, len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } } else { if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { @@ -1158,6 +1166,10 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_Segment *seg = MPID_Segment_alloc(); MPI_Aint last = type_size * get_accum_pkt->count; + if (seg == NULL) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + } MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment"); MPID_Segment_init(get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, seg, 0); @@ -1170,19 +1182,19 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc /* All data fits in packet header */ mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { MPIU_Assert(get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM); mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + /* here we increment the Active Target counter to guarantee the GET-like operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; @@ -1295,7 +1307,11 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* copy data to resp pkt header */ void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; mpi_errno = immed_copy(src, dest, type_size); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } } else { MPIU_Memcpy(resp_req->dev.user_buf, fop_pkt->addr, @@ -1312,12 +1328,13 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en mpi_errno = do_accumulate_op(lock_entry->data, fop_pkt->addr, 1, fop_pkt->datatype, fop_pkt->op); } - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,lock_entry->vc); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 1421224..0f4674e 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -812,7 +812,11 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* copy data from target buffer to response packet header */ src = (void *) (get_accum_pkt->addr), dest = (void *) (get_accum_resp_pkt->info.data); mpi_errno = immed_copy(src, dest, len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } } else { MPIU_Memcpy(resp_req->dev.user_buf, get_accum_pkt->addr, @@ -823,13 +827,12 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); @@ -1209,20 +1212,23 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* copy data to resp pkt header */ void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; mpi_errno = immed_copy(src, dest, type_size); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { mpi_errno = do_accumulate_op(fop_pkt->info.data, fop_pkt->addr, 1, fop_pkt->datatype, fop_pkt->op); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_POP(mpi_errno); - } } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); http://git.mpich.org/mpich.git/commitdiff/c8ecef8d531601900e9d8db58284897711... commit c8ecef8d531601900e9d8db58284897711295d12 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 10:24:24 2015 -0800 Bug-fix: use do_accumulate_op function for ACC computation. do_accumulate_op() does more comprehensive work on ACC computation than OP function. For example, MPI_REPLACE is not defined as predefined computation and therefore not handled by OP function, but it is safely handled in do_accumulate_op(). This patch replace OP function with do_accumulate_op() on target side. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index c6fc623..5943d35 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -1304,14 +1304,15 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { - MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); - int one = 1; if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { - (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + mpi_errno = do_accumulate_op(fop_pkt->info.data, fop_pkt->addr, + 1, fop_pkt->datatype, fop_pkt->op); } else { - (*uop)(lock_entry->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + mpi_errno = do_accumulate_op(lock_entry->data, fop_pkt->addr, + 1, fop_pkt->datatype, fop_pkt->op); } + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 1f70002..1421224 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -1213,9 +1213,11 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { - MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); - int one = 1; - (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + mpi_errno = do_accumulate_op(fop_pkt->info.data, fop_pkt->addr, + 1, fop_pkt->datatype, fop_pkt->op); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } } if (win_ptr->shm_allocated == TRUE) http://git.mpich.org/mpich.git/commitdiff/59afc29cbf253c7cb7f8efb052e816338c... commit 59afc29cbf253c7cb7f8efb052e816338cb99304 Author: Xin Zhao <[email protected]> Date: Wed Jan 28 10:07:24 2015 -0600 Use memcpy for structure assignment. In this patch we replace "=" with memcpy function when assigning structure content to another struct. Using "=" in this case is not compatible for llvm compiler. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_lockqueue.h b/src/mpid/ch3/include/mpid_rma_lockqueue.h index 1c98ced..13f1290 100644 --- a/src/mpid/ch3/include/mpid_rma_lockqueue.h +++ b/src/mpid/ch3/include/mpid_rma_lockqueue.h @@ -31,7 +31,7 @@ static inline MPIDI_RMA_Lock_entry_t *MPIDI_CH3I_Win_lock_entry_alloc(MPID_Win * if (new_ptr != NULL) { new_ptr->next = NULL; - new_ptr->pkt = (*pkt); + MPIU_Memcpy(&(new_ptr->pkt), pkt, sizeof(*pkt)); new_ptr->vc = NULL; new_ptr->data = NULL; new_ptr->data_size = 0; http://git.mpich.org/mpich.git/commitdiff/1b30ab19a3dfdd6ae31c87dba1928ee68e... commit 1b30ab19a3dfdd6ae31c87dba1928ee68ec2775a Author: Xin Zhao <[email protected]> Date: Tue Feb 10 10:19:40 2015 -0800 Change argument of function finish_op_on_target. In this patch, we replace one argument of function finish_op_on_target, "packet(op) type", with "has_response_data". Since finish_op_on_target does not care what specific packet(op) type it is processing on, but only cares about if the current op has response data (like GET/GACC), changing the argument in this way can simplify the code by avoiding acquiring packet(op) type everytime before calling finish_op_on_target. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index d35e5b2..1b4fdf9 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -851,13 +851,12 @@ static inline int check_piggyback_lock(MPID_Win *win_ptr, MPIDI_VC_t *vc, } static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, - MPIDI_CH3_Pkt_type_t type, + int has_response_data, MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle) { int mpi_errno = MPI_SUCCESS; - if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_PUT_IMMED || - type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || type == MPIDI_CH3_PKT_ACCUMULATE) { + if (!has_response_data) { /* This is PUT or ACC */ if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index fbb38ab..c6fc623 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -113,7 +113,7 @@ int MPIDI_CH3_ReqHandler_PutRecvComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_PUT, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, flags, source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -191,7 +191,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_ACCUMULATE, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, flags, source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -957,7 +957,7 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, put_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, FALSE /* has no response data */, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1087,7 +1087,7 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, acc_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, FALSE /* has no response data */, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1360,7 +1360,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, fop_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1442,7 +1442,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, cas_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c index 84ad691..93d8e5d 100644 --- a/src/mpid/ch3/src/ch3u_handle_send_req.c +++ b/src/mpid/ch3/src/ch3u_handle_send_req.c @@ -84,7 +84,7 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_GET, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -149,7 +149,7 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_GET_ACCUM, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -217,7 +217,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS_IMMED, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE/* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -283,7 +283,7 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 4163baa..1f70002 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -225,7 +225,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_Memcpy(put_pkt->addr, put_pkt->info.data, put_pkt->count*type_size); /* trigger final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -577,7 +577,7 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } /* trigger final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, accum_pkt->flags, accum_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1074,7 +1074,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Request_release(req); } - mpi_errno = finish_op_on_target(win_ptr, vc, cas_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1247,7 +1247,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } } - mpi_errno = finish_op_on_target(win_ptr, vc, fop_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } http://git.mpich.org/mpich.git/commitdiff/21479b00b1080d577be8ff7ed66a7bdd62... commit 21479b00b1080d577be8ff7ed66a7bdd62dd16ac Author: Xin Zhao <[email protected]> Date: Mon Feb 9 22:35:28 2015 -0800 Add asserts for RMA packet types. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index ea1036d..fbb38ab 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -936,6 +936,12 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_put_t *put_pkt = &((lock_entry->pkt).put); int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + if (put_pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { /* all data fits in packet header */ mpi_errno = MPIR_Localcopy(put_pkt->info.data, put_pkt->count, put_pkt->datatype, @@ -973,6 +979,12 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + sreq = MPID_Request_create(); if (sreq == NULL) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); @@ -1052,6 +1064,9 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIU_Assert(lock_entry->all_data_recved == 1); + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(acc_pkt->datatype)); + if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); @@ -1095,6 +1110,12 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + sreq = MPID_Request_create(); if (sreq == NULL) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); @@ -1218,6 +1239,12 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en int iovcnt; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(fop_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + /* FIXME: this function is same with PktHandler_FOP(), should do code refactoring on both of them. */ @@ -1353,6 +1380,12 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPI_Aint len; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(cas_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; http://git.mpich.org/mpich.git/commitdiff/de9d0f21b33691ee555d3479acdc5e5863... commit de9d0f21b33691ee555d3479acdc5e5863af92ca Author: Xin Zhao <[email protected]> Date: Mon Feb 9 19:14:32 2015 -0800 Rewrite code of piggybacking IMMED data with RMA packets. Originally we add "immed_data" and "immed_len" areas to RMA packets, in order to piggyback small amount of data with packet header to reduce number of packets (Note that "immed_len" is necessary when the piggybacked data is not the entire data). However, those areas potentially increase the packet union size and worsen the two-sided communication. This patch fixes this issue. In this patch, we remove "immed_data" and "immed_len" from normal "MPIDI_CH3_Pkt_XXX_t" operation type (e.g. MPIDI_CH3_Pkt_put_t), and we introduce new "MPIDI_CH3_Pkt_XXX_immed_t" packt type for each operation (e.g. MPIDI_CH3_Pkt_put_immed_t). "MPIDI_CH3_Pkt_XXX_immed_t" is used when (1) both origin and target are basic datatypes, AND, (2) the data to be sent can be entirely fit into the header. By doing this, "MPIDI_CH3_Pkt_XXX_immed_t" needs "immed_data" area but can drop "immed_len" area. Also, since it only works with basic target datatype, it can drop "dataloop_size" area as well. All operations that do not satisfy (1) or (2) will use normal "MPIDI_CH3_Pkt_XXX_t" type. Originally we always piggyback FOP data into the packet header, which makes the packet size too large. In this patch we split the FOP operaton into IMMED packets and normal packets. Because CAS only work with 2 basic datatype and non-complex elements, the data amount is relatively small, we always piggyback the data with packet header and only use "MPIDI_CH3_Pkt_XXX_immed_t" packet type for CAS. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index a443521..2c1a821 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -326,7 +326,6 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags) { MPIDI_VC_t *vc = NULL; - size_t len; MPI_Aint origin_type_size; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_put_t *put_pkt = &rma_op->pkt.put; @@ -341,23 +340,10 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, put_pkt->flags |= flags; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - if (!rma_op->is_dt) { - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(put_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (put_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = put_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)put_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (len == (size_t)put_pkt->immed_len) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, put_pkt, sizeof(*put_pkt), &(rma_op->request)); @@ -368,10 +354,8 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, /* We still need to issue from origin buffer. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt; iov[0].MPID_IOV_LEN = sizeof(*put_pkt); - if (!rma_op->is_dt) { - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr + put_pkt->immed_len); - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size - put_pkt->immed_len; - } + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -397,7 +381,6 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags) { MPIDI_VC_t *vc = NULL; - size_t len; MPI_Aint origin_type_size; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_accum_t *accum_pkt = &rma_op->pkt.accum; @@ -412,23 +395,10 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, accum_pkt->flags |= flags; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - if (!rma_op->is_dt) { - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(accum_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (accum_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = accum_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)accum_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (len == (size_t)accum_pkt->immed_len) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, accum_pkt, sizeof(*accum_pkt), &(rma_op->request)); @@ -439,10 +409,8 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, /* We still need to issue from origin buffer. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); - if (!rma_op->is_dt) { - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr + accum_pkt->immed_len); - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size - accum_pkt->immed_len; - } + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -466,7 +434,6 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags) { MPIDI_VC_t *vc = NULL; - size_t len; MPI_Aint origin_type_size; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt = &rma_op->pkt.get_accum; @@ -505,30 +472,12 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, get_accum_pkt->request_handle = resp_req->handle; get_accum_pkt->flags |= flags; - if (!rma_op->is_dt) { - /* Only fill IMMED data in response packet when both origin and target - buffers are basic datatype. */ - get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; - } MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - if (!rma_op->is_dt) { - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(get_accum_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (get_accum_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = get_accum_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)get_accum_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (len == (size_t)get_accum_pkt->immed_len) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, get_accum_pkt, sizeof(*get_accum_pkt), &(rma_op->request)); @@ -539,10 +488,8 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, /* We still need to issue from origin buffer. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_pkt); - if (!rma_op->is_dt) { - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr + get_accum_pkt->immed_len); - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size - get_accum_pkt->immed_len; - } + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -636,11 +583,6 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, get_pkt->request_handle = rma_op->request->handle; get_pkt->flags |= flags; - if (!rma_op->is_dt) { - /* Only fill IMMED data in response packet when both origin and target - buffers are basic datatype. */ - get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; - } comm_ptr = win_ptr->comm_ptr; MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); @@ -705,7 +647,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_RMA_Target_t *target_ptr, MPIDI_CH3_Pkt_flags_t flags) { - MPI_Aint len; MPIDI_VC_t *vc = NULL; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_cas_t *cas_pkt = &rma_op->pkt.cas; @@ -729,20 +670,12 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, rma_op->request->dev.user_count = rma_op->result_count; rma_op->request->dev.datatype = rma_op->result_datatype; - /* REQUIRE: All datatype arguments must be of the same, builtin - * type and counts must be 1. */ - MPID_Datatype_get_size_macro(rma_op->origin_datatype, len); - MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); - rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle; cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags |= flags; - MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); - MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_pkt, sizeof(*cas_pkt), &rmw_req); @@ -782,8 +715,8 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_fop_t *fop_pkt = &rma_op->pkt.fop; MPID_Request *resp_req = NULL; - size_t len; MPI_Aint origin_type_size; + MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_ISSUE_FOP_OP); @@ -809,26 +742,27 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, fop_pkt->flags |= flags; - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(fop_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (fop_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = fop_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)fop_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); + + if (rma_op->pkt.type == MPIDI_CH3_PKT_FOP_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_pkt, sizeof(*fop_pkt), &(rma_op->request)); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + else { + /* We still need to issue from origin buffer. */ + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = origin_type_size; + + mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } /* This operation can generate two requests; one for inbound and one for * outbound data. */ @@ -888,21 +822,25 @@ static inline int issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, switch (op_ptr->pkt.type) { case (MPIDI_CH3_PKT_PUT): + case (MPIDI_CH3_PKT_PUT_IMMED): mpi_errno = issue_put_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_ACCUMULATE): + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): mpi_errno = issue_acc_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_GET_ACCUM): + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): mpi_errno = issue_get_acc_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_GET): mpi_errno = issue_get_op(op_ptr, win_ptr, target_ptr, flags); break; - case (MPIDI_CH3_PKT_CAS): + case (MPIDI_CH3_PKT_CAS_IMMED): mpi_errno = issue_cas_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_FOP): + case (MPIDI_CH3_PKT_FOP_IMMED): mpi_errno = issue_fop_op(op_ptr, win_ptr, target_ptr, flags); break; default: diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 1c924be..6b02880 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1913,6 +1913,8 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *, MPID_Request *, int * ); int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *, MPID_Request *, int * ); +int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *, MPID_Request *, + int * ); int MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete( MPIDI_VC_t *, MPID_Request *, int * ); diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index f6e5327..cbd300c 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -76,15 +76,22 @@ typedef enum { MPIDI_CH3_PKT_CANCEL_SEND_RESP, /* RMA Packets begin here */ MPIDI_CH3_PKT_PUT, + MPIDI_CH3_PKT_PUT_IMMED, MPIDI_CH3_PKT_GET, MPIDI_CH3_PKT_ACCUMULATE, + MPIDI_CH3_PKT_ACCUMULATE_IMMED, MPIDI_CH3_PKT_GET_ACCUM, + MPIDI_CH3_PKT_GET_ACCUM_IMMED, MPIDI_CH3_PKT_FOP, - MPIDI_CH3_PKT_CAS, + MPIDI_CH3_PKT_FOP_IMMED, + MPIDI_CH3_PKT_CAS_IMMED, MPIDI_CH3_PKT_GET_RESP, + MPIDI_CH3_PKT_GET_RESP_IMMED, MPIDI_CH3_PKT_GET_ACCUM_RESP, + MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED, MPIDI_CH3_PKT_FOP_RESP, - MPIDI_CH3_PKT_CAS_RESP, + MPIDI_CH3_PKT_FOP_RESP_IMMED, + MPIDI_CH3_PKT_CAS_RESP_IMMED, MPIDI_CH3_PKT_LOCK, MPIDI_CH3_PKT_LOCK_ACK, MPIDI_CH3_PKT_LOCK_OP_ACK, @@ -198,21 +205,25 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ datatype_ = (pkt_).put.datatype; \ break; \ case (MPIDI_CH3_PKT_GET): \ datatype_ = (pkt_).get.datatype; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ datatype_ = (pkt_).accum.datatype; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ datatype_ = (pkt_).get_accum.datatype; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ datatype_ = (pkt_).cas.datatype; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ datatype_ = (pkt_).fop.datatype; \ break; \ default: \ @@ -227,19 +238,23 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ count_ = (pkt_).put.count; \ break; \ case (MPIDI_CH3_PKT_GET): \ count_ = (pkt_).get.count; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ count_ = (pkt_).accum.count; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ count_ = (pkt_).get_accum.count; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ count_ = 1; \ break; \ default: \ @@ -247,47 +262,6 @@ MPIDI_CH3_PKT_DEFS } \ } -#define MPIDI_CH3_PKT_RMA_GET_IMMED_LEN(pkt_, immed_len_, err_) \ - { \ - /* This macro returns immed_len in RMA operation \ - packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ - packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ - err_ = MPI_SUCCESS; \ - switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - immed_len_ = (pkt_).put.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - immed_len_ = (pkt_).accum.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - immed_len_ = (pkt_).get_accum.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - immed_len_ = (pkt_).fop.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_CAS): \ - /* Note that here we return size of origin data, not \ - size of compare data. */ \ - immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ - break; \ - case (MPIDI_CH3_PKT_GET_RESP): \ - immed_len_ = (pkt_).get_resp.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ - immed_len_ = (pkt_).get_accum_resp.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_FOP_RESP): \ - immed_len_ = (pkt_).fop_resp.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ - immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ - break; \ - default: \ - MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ - } \ - } - #define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_) \ { \ /* This macro returns pointer to immed data in RMA operation \ @@ -295,34 +269,34 @@ MPIDI_CH3_PKT_DEFS packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - immed_data_ = (pkt_).put.data; \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ + immed_data_ = (pkt_).put.info.data; \ break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - immed_data_ = (pkt_).accum.data; \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ + immed_data_ = (pkt_).accum.info.data; \ break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - immed_data_ = (pkt_).get_accum.data; \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ + immed_data_ = (pkt_).get_accum.info.data; \ break; \ - case (MPIDI_CH3_PKT_FOP): \ - immed_data_ = (pkt_).fop.data; \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ + immed_data_ = (pkt_).fop.info.data; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ /* Note that here we return pointer of origin data, not \ pointer of compare data. */ \ immed_data_ = &((pkt_).cas.origin_data); \ break; \ - case (MPIDI_CH3_PKT_GET_RESP): \ - immed_data_ = (pkt_).get_resp.data; \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ + immed_data_ = (pkt_).get_resp.info.data; \ break; \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ - immed_data_ = (pkt_).get_accum_resp.data; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ + immed_data_ = (pkt_).get_accum_resp.info.data; \ break; \ - case (MPIDI_CH3_PKT_FOP_RESP): \ - immed_data_ = (pkt_).fop_resp.data; \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ + immed_data_ = (pkt_).fop_resp.info.data; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ - immed_data_ = &((pkt_).cas_resp.data); \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ + immed_data_ = &((pkt_).cas_resp.info.data); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ @@ -339,33 +313,40 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ flags_ = (pkt_).put.flags; \ break; \ case (MPIDI_CH3_PKT_GET): \ flags_ = (pkt_).get.flags; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ flags_ = (pkt_).accum.flags; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ flags_ = (pkt_).get_accum.flags; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ flags_ = (pkt_).cas.flags; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ flags_ = (pkt_).fop.flags; \ break; \ case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ flags_ = (pkt_).get_resp.flags; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ flags_ = (pkt_).get_accum_resp.flags; \ break; \ case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ flags_ = (pkt_).fop_resp.flags; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ flags_ = (pkt_).cas_resp.flags; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -395,33 +376,40 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET): \ (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -451,33 +439,40 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ win_hdl_ = (pkt_).put.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET): \ win_hdl_ = (pkt_).get.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ win_hdl_ = (pkt_).accum.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ win_hdl_ = (pkt_).get_accum.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ win_hdl_ = (pkt_).cas.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ win_hdl_ = (pkt_).fop.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ win_hdl_ = (pkt_).get_resp.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ win_hdl_ = (pkt_).get_accum_resp.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ win_hdl_ = (pkt_).fop_resp.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ win_hdl_ = (pkt_).cas_resp.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -511,21 +506,25 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ win_hdl_ = (pkt_).put.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET): \ win_hdl_ = (pkt_).get.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ win_hdl_ = (pkt_).accum.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ win_hdl_ = (pkt_).get_accum.target_win_handle; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ win_hdl_ = (pkt_).cas.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ win_hdl_ = (pkt_).fop.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -552,30 +551,28 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ - (pkt_).put.dataloop_size = (dataloop_size_); \ + (pkt_).put.info.dataloop_size = (dataloop_size_); \ break; \ case (MPIDI_CH3_PKT_GET): \ - (pkt_).get.dataloop_size = (dataloop_size_); \ + (pkt_).get.info.dataloop_size = (dataloop_size_); \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ - (pkt_).accum.dataloop_size = (dataloop_size_); \ + (pkt_).accum.info.dataloop_size = (dataloop_size_); \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ - (pkt_).get_accum.dataloop_size = (dataloop_size_); \ + (pkt_).get_accum.info.dataloop_size = (dataloop_size_); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ } - typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -583,8 +580,10 @@ typedef struct MPIDI_CH3_Pkt_put { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + union { + int dataloop_size; + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_put_t; typedef struct MPIDI_CH3_Pkt_get { @@ -593,7 +592,12 @@ typedef struct MPIDI_CH3_Pkt_get { void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ + struct { + /* note that we use struct here in order + to consistently access dataloop_size + by "pkt->info.dataloop_size". */ + int dataloop_size; /* for derived datatypes */ + } info; MPI_Request request_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in @@ -612,8 +616,12 @@ typedef struct MPIDI_CH3_Pkt_get_resp { MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_get_resp_t; typedef struct MPIDI_CH3_Pkt_accum { @@ -622,7 +630,6 @@ typedef struct MPIDI_CH3_Pkt_accum { void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ MPI_Op op; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in @@ -631,8 +638,10 @@ typedef struct MPIDI_CH3_Pkt_accum { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + union { + int dataloop_size; + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum { @@ -642,7 +651,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum { void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ MPI_Op op; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in @@ -651,8 +659,10 @@ typedef struct MPIDI_CH3_Pkt_get_accum { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + union { + int dataloop_size; + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_get_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum_resp { @@ -663,8 +673,12 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_cas { @@ -685,7 +699,12 @@ typedef struct MPIDI_CH3_Pkt_cas { typedef struct MPIDI_CH3_Pkt_cas_resp { MPIDI_CH3_Pkt_type_t type; MPI_Request request_handle; - MPIDI_CH3_CAS_Immed_u data; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + MPIDI_CH3_CAS_Immed_u data; + } info; /* followings are used to decrement ack_counter at orign */ int target_rank; MPI_Win source_win_handle; @@ -704,15 +723,23 @@ typedef struct MPIDI_CH3_Pkt_fop { * epoch for decrementing rma op counter in * active target rma and for unlocking window * in passive target rma. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_fop_t; typedef struct MPIDI_CH3_Pkt_fop_resp { MPIDI_CH3_Pkt_type_t type; MPI_Request request_handle; - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; /* followings are used to decrement ack_counter at orign */ int target_rank; MPI_Win source_win_handle; diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 2bb62b2..9c2618d 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -433,7 +433,6 @@ typedef struct MPIDI_Request { MPI_Op op; /* For accumulate, since data is first read into a tmp_buf */ void *real_user_buf; - void *final_user_buf; /* For derived datatypes at target */ struct MPIDI_RMA_dtype_info *dtype_info; void *dataloop; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index d7a720d..d35e5b2 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -333,9 +333,12 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, } if (pkt->type == MPIDI_CH3_PKT_LOCK || + pkt->type == MPIDI_CH3_PKT_PUT_IMMED || + pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || pkt->type == MPIDI_CH3_PKT_GET || - pkt->type == MPIDI_CH3_PKT_FOP || - pkt->type == MPIDI_CH3_PKT_CAS) { + pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || + pkt->type == MPIDI_CH3_PKT_FOP_IMMED || + pkt->type == MPIDI_CH3_PKT_CAS_IMMED) { /* return bytes of data processed in this pkt handler */ (*buflen) = sizeof(MPIDI_CH3_Pkt_t); @@ -351,8 +354,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPID_Request *req = NULL; MPI_Datatype target_dtp; int target_count; - int immed_len = 0; - void *immed_data = NULL; int complete = 0; MPIDI_msg_sz_t data_len; char *data_buf = NULL; @@ -365,17 +366,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPID_Datatype_get_size_macro(target_dtp, type_size); recv_data_sz = type_size * target_count; - if (recv_data_sz <= MPIDI_RMA_IMMED_BYTES) { - - /* return bytes of data processed in this pkt handler */ - (*buflen) = sizeof(MPIDI_CH3_Pkt_t); - - if (new_ptr != NULL) - new_ptr->all_data_recved = 1; - - goto issue_ack; - } - if (new_ptr != NULL) { if (win_ptr->current_lock_data_bytes + recv_data_sz < MPIR_CVAR_CH3_RMA_LOCK_DATA_BYTES) { @@ -430,10 +420,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, req->dev.OnFinal = MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete; req->dev.lock_queue_entry = new_ptr; - MPIDI_CH3_PKT_RMA_GET_IMMED_LEN((*pkt), immed_len, mpi_errno); - if (immed_len > 0) { - req->dev.recv_data_sz -= immed_len; - } data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); MPIU_Assert(req->dev.recv_data_sz > 0); @@ -447,15 +433,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, req->dev.OnFinal = MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete; req->dev.lock_queue_entry = new_ptr; - MPIDI_CH3_PKT_RMA_GET_IMMED_LEN((*pkt), immed_len, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR((*pkt), immed_data, mpi_errno); - - if (immed_len > 0) { - /* see if we can receive some data from packet header */ - MPIU_Memcpy(req->dev.user_buf, immed_data, (size_t)immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + immed_len); - req->dev.recv_data_sz -= immed_len; - } data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); MPIU_Assert(req->dev.recv_data_sz > 0); @@ -597,7 +574,9 @@ static inline int adjust_op_piggybacked_with_lock (MPID_Win *win_ptr, &(target->dt_op_list_tail), op); } else if (op->pkt.type == MPIDI_CH3_PKT_PUT || - op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || + op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || + op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list), &(target->write_op_list_tail), op); } @@ -877,7 +856,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPI_Win source_win_handle) { int mpi_errno = MPI_SUCCESS; - if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_ACCUMULATE) { + if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_PUT_IMMED || + type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || type == MPIDI_CH3_PKT_ACCUMULATE) { /* This is PUT or ACC */ if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { diff --git a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c index c2ae4a7..919d097 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c @@ -573,12 +573,18 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[], We could even do lazy initialization (make this part of win_create) */ pktArray[MPIDI_CH3_PKT_PUT] = MPIDI_CH3_PktHandler_Put; + pktArray[MPIDI_CH3_PKT_PUT_IMMED] = + MPIDI_CH3_PktHandler_Put; pktArray[MPIDI_CH3_PKT_ACCUMULATE] = MPIDI_CH3_PktHandler_Accumulate; + pktArray[MPIDI_CH3_PKT_ACCUMULATE_IMMED] = + MPIDI_CH3_PktHandler_Accumulate; pktArray[MPIDI_CH3_PKT_GET] = MPIDI_CH3_PktHandler_Get; pktArray[MPIDI_CH3_PKT_GET_RESP] = MPIDI_CH3_PktHandler_GetResp; + pktArray[MPIDI_CH3_PKT_GET_RESP_IMMED] = + MPIDI_CH3_PktHandler_GetResp; pktArray[MPIDI_CH3_PKT_LOCK] = MPIDI_CH3_PktHandler_Lock; pktArray[MPIDI_CH3_PKT_LOCK_ACK] = @@ -593,18 +599,26 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[], MPIDI_CH3_PktHandler_FlushAck; pktArray[MPIDI_CH3_PKT_DECR_AT_COUNTER] = MPIDI_CH3_PktHandler_DecrAtCnt; - pktArray[MPIDI_CH3_PKT_CAS] = + pktArray[MPIDI_CH3_PKT_CAS_IMMED] = MPIDI_CH3_PktHandler_CAS; - pktArray[MPIDI_CH3_PKT_CAS_RESP] = + pktArray[MPIDI_CH3_PKT_CAS_RESP_IMMED] = MPIDI_CH3_PktHandler_CASResp; pktArray[MPIDI_CH3_PKT_FOP] = MPIDI_CH3_PktHandler_FOP; + pktArray[MPIDI_CH3_PKT_FOP_IMMED] = + MPIDI_CH3_PktHandler_FOP; pktArray[MPIDI_CH3_PKT_FOP_RESP] = MPIDI_CH3_PktHandler_FOPResp; + pktArray[MPIDI_CH3_PKT_FOP_RESP_IMMED] = + MPIDI_CH3_PktHandler_FOPResp; pktArray[MPIDI_CH3_PKT_GET_ACCUM] = MPIDI_CH3_PktHandler_GetAccumulate; + pktArray[MPIDI_CH3_PKT_GET_ACCUM_IMMED] = + MPIDI_CH3_PktHandler_GetAccumulate; pktArray[MPIDI_CH3_PKT_GET_ACCUM_RESP] = MPIDI_CH3_PktHandler_Get_AccumResp; + pktArray[MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED] = + MPIDI_CH3_PktHandler_Get_AccumResp; /* End of default RMA operations */ /* Fault tolerance */ diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 302e167..ea1036d 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -172,7 +172,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc, if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); /* accumulate data from tmp_buf into user_buf */ - mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf, + mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); @@ -182,7 +182,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc, /* free the temporary buffer */ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); - MPIU_Free((char *) rreq->dev.final_user_buf + true_lb); + MPIU_Free((char *) rreq->dev.user_buf + true_lb); /* mark data transfer as complete and decrement CC */ MPIDI_CH3U_Request_complete(rreq); @@ -224,7 +224,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Request *resp_req; MPID_IOV iov[MPID_IOV_LIMIT]; MPI_Aint true_lb, true_extent; - size_t len; int iovcnt; MPIU_CHKPMEM_DECL(1); MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE); @@ -233,7 +232,12 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); + if ((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); + } + else { MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); + } get_accum_resp_pkt->request_handle = rreq->dev.resp_request_handle; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_accum_resp_pkt->source_win_handle = rreq->dev.source_win_handle; @@ -244,7 +248,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - get_accum_resp_pkt->immed_len = 0; MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size); @@ -253,12 +256,22 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); MPIU_Object_set_ref(resp_req, 1); + if (!((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, rreq->dev.user_count * type_size, mpi_errno, "GACC resp. buffer"); + } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if ((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + void *src = (void *)(rreq->dev.real_user_buf), *dest = (void *)(get_accum_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, rreq->dev.user_count * type_size); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + } + else { if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) { MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count * type_size); @@ -271,9 +284,10 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Segment_pack(seg, 0, &last, resp_req->dev.user_buf); MPID_Segment_free(seg); } + } /* accumulate data from tmp_buf into user_buf */ - mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf, + mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -291,27 +305,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; - /* length of target data */ - MPIU_Assign_trunc(len, rreq->dev.user_count * type_size, size_t); - - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ - if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_accum_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_accum_resp_pkt->immed_len > 0) { - void *src = resp_req->dev.user_buf; - void *dest = (void*) get_accum_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_accum_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } - - if (len == (size_t)get_accum_resp_pkt->immed_len) { + if ((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); @@ -320,8 +314,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf + get_accum_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = rreq->dev.user_count * type_size - get_accum_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = rreq->dev.user_count * type_size; iovcnt = 2; } @@ -338,7 +332,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, /* free the temporary buffer */ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); - MPIU_Free((char *) rreq->dev.final_user_buf + true_lb); + MPIU_Free((char *) rreq->dev.user_buf + true_lb); /* mark data transfer as complete and decrement CC */ MPIDI_CH3U_Request_complete(rreq); @@ -355,6 +349,112 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, /* --END ERROR HANDLING-- */ } + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_ReqHandler_FOPRecvComplete +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *vc, + MPID_Request *rreq, + int *complete ) +{ + int mpi_errno = MPI_SUCCESS; + MPID_Win *win_ptr = NULL; + MPI_Aint type_size; + MPID_Request *resp_req = NULL; + MPID_IOV iov[MPID_IOV_LIMIT]; + int iovcnt; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; + MPIU_CHKPMEM_DECL(1); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + + MPIU_Assert(!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + + MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); + + MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size); + + /* Create response request */ + resp_req = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + MPIU_Object_set_ref(resp_req, 1); + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_FOPSendComplete; + resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPSendComplete; + resp_req->dev.target_win_handle = rreq->dev.target_win_handle; + resp_req->dev.flags = rreq->dev.flags; + + MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, type_size, + mpi_errno, "FOP resp. buffer"); + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + + /* Copy data into a temporary buffer in response request */ + MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, type_size); + + /* Perform accumulate computation */ + if (rreq->dev.op != MPI_NO_OP) { + mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, + 1, rreq->dev.datatype, rreq->dev.op); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + + /* Send back data */ + MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); + fop_resp_pkt->request_handle = rreq->dev.resp_request_handle; + fop_resp_pkt->source_win_handle = rreq->dev.source_win_handle; + fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; + fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) + fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; + if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || + (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) + fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_resp_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = type_size; + iovcnt = 2; + + MPIU_THREAD_CS_ENTER(CH3COMM,vc); + mpi_errno = MPIDI_CH3_iSendv(vc, resp_req, iov, iovcnt); + MPIU_THREAD_CS_EXIT(CH3COMM,vc); + + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + /* free the temporary buffer */ + MPIU_Free((char *) rreq->dev.user_buf); + + /* mark data transfer as complete and decrement CC */ + MPIDI_CH3U_Request_complete(rreq); + *complete = TRUE; + + fn_exit: + MPIU_CHKPMEM_COMMIT(); + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + return MPI_SUCCESS; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + #undef FUNCNAME #define FUNCNAME MPIDI_CH3_ReqHandler_PutDerivedDTRecvComplete #undef FCNAME @@ -444,7 +544,6 @@ int MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete( MPIDI_VC_t *vc ATTRIBUTE((u tmp_buf = (void *)((char*)tmp_buf - true_lb); rreq->dev.user_buf = tmp_buf; - rreq->dev.final_user_buf = rreq->dev.user_buf; rreq->dev.datatype = new_dtp->handle; rreq->dev.recv_data_sz = new_dtp->size * rreq->dev.user_count; @@ -494,6 +593,8 @@ int MPIDI_CH3_ReqHandler_GaccumDerivedDTRecvComplete( MPIDI_VC_t *vc ATTRIBUTE(( MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMDERIVEDDTRECVCOMPLETE); + MPIU_Assert(!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + /* create derived datatype */ create_derived_datatype(rreq, &new_dtp); @@ -516,7 +617,6 @@ int MPIDI_CH3_ReqHandler_GaccumDerivedDTRecvComplete( MPIDI_VC_t *vc ATTRIBUTE(( tmp_buf = (void *)((char*)tmp_buf - true_lb); rreq->dev.user_buf = tmp_buf; - rreq->dev.final_user_buf = rreq->dev.user_buf; rreq->dev.datatype = new_dtp->handle; rreq->dev.recv_data_sz = new_dtp->size * rreq->dev.user_count; @@ -571,6 +671,8 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); + MPIU_Assert(!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + /* create derived datatype */ create_derived_datatype(rreq, &new_dtp); MPIU_Free(rreq->dev.dtype_info); @@ -602,7 +704,6 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - get_resp_pkt->immed_len = 0; sreq->dev.segment_ptr = MPID_Segment_alloc( ); MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc"); @@ -835,20 +936,22 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_put_t *put_pkt = &((lock_entry->pkt).put); int mpi_errno = MPI_SUCCESS; - if (lock_entry->data == NULL) { + if (put_pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { /* all data fits in packet header */ - mpi_errno = MPIR_Localcopy(put_pkt->data, put_pkt->count, put_pkt->datatype, + mpi_errno = MPIR_Localcopy(put_pkt->info.data, put_pkt->count, put_pkt->datatype, put_pkt->addr, put_pkt->count, put_pkt->datatype); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { + MPIU_Assert(put_pkt->type == MPIDI_CH3_PKT_PUT); + mpi_errno = MPIR_Localcopy(lock_entry->data, put_pkt->count, put_pkt->datatype, put_pkt->addr, put_pkt->count, put_pkt->datatype); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_PUT, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, put_pkt->type, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -888,7 +991,12 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP_IMMED); + } + else { MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + } get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -899,30 +1007,18 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_resp_pkt->source_win_handle = get_pkt->source_win_handle; - get_resp_pkt->immed_len = 0; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t); - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_resp_pkt->immed_len > 0) { - void *src = get_pkt->addr; - void *dest = (void*) get_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } + void *src = (void *)(get_pkt->addr), *dest = (void *)(get_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - if (len == (size_t)get_resp_pkt->immed_len) { + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); @@ -931,8 +1027,8 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)get_pkt->addr + get_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = get_pkt->count * type_size - get_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) (get_pkt->addr); + iov[1].MPID_IOV_LEN = get_pkt->count * type_size; iovcnt = 2; } @@ -959,13 +1055,15 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - if (lock_entry->data == NULL) { + if (acc_pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { /* All data fits in packet header */ - mpi_errno = do_accumulate_op(acc_pkt->data, acc_pkt->addr, + mpi_errno = do_accumulate_op(acc_pkt->info.data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { + MPIU_Assert(acc_pkt->type == MPIDI_CH3_PKT_ACCUMULATE); + mpi_errno = do_accumulate_op(lock_entry->data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -974,7 +1072,7 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_ACCUMULATE, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, acc_pkt->type, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1013,12 +1111,25 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc /* Copy data into a temporary buffer */ MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); + if (!(get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) sreq->dev.user_buf = (void *)MPIU_Malloc(get_accum_pkt->count * type_size); + else { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); + } + + /* length of target data */ + MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); /* Perform ACCUMULATE OP */ if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + void *src = (void *)(get_accum_pkt->addr), *dest = (void *)(get_accum_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { MPIU_Memcpy(sreq->dev.user_buf, get_accum_pkt->addr, get_accum_pkt->count * type_size); @@ -1032,14 +1143,17 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_Segment_pack(seg, 0, &last, sreq->dev.user_buf); MPID_Segment_free(seg); } + } - if (lock_entry->data == NULL) { + if (get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { /* All data fits in packet header */ - mpi_errno = do_accumulate_op(get_accum_pkt->data, get_accum_pkt->addr, + mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { + MPIU_Assert(get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM); + mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1052,7 +1166,9 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; + if (!(get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); + } get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1063,29 +1179,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; - get_accum_resp_pkt->immed_len = 0; - - /* length of target data */ - MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_accum_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_accum_resp_pkt->immed_len > 0) { - void *src = sreq->dev.user_buf; - void *dest = (void*) get_accum_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_accum_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } - - if (len == (size_t)get_accum_resp_pkt->immed_len) { /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); @@ -1094,8 +1189,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)sreq->dev.user_buf + get_accum_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = get_accum_pkt->count * type_size - get_accum_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)sreq->dev.user_buf); + iov[1].MPID_IOV_LEN = get_accum_pkt->count * type_size; iovcnt = 2; } @@ -1118,12 +1213,23 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; MPIDI_CH3_Pkt_fop_t *fop_pkt = &((lock_entry->pkt).fop); MPID_Request *resp_req = NULL; + MPI_Aint type_size; + MPID_IOV iov[MPID_IOV_LIMIT]; + int iovcnt; int mpi_errno = MPI_SUCCESS; /* FIXME: this function is same with PktHandler_FOP(), should do code refactoring on both of them. */ + MPID_Datatype_get_size_macro(fop_pkt->datatype, type_size); + + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED); + } + else { MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); + } + fop_resp_pkt->request_handle = fop_pkt->request_handle; fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -1134,26 +1240,57 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - fop_resp_pkt->immed_len = fop_pkt->immed_len; + + if (!(fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { + resp_req = MPID_Request_create(); + if (resp_req == NULL) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + } + MPIU_Object_set_ref(resp_req, 1); + + resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPSendComplete; + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_FOPSendComplete; + + resp_req->dev.target_win_handle = win_ptr->handle; + resp_req->dev.flags = fop_pkt->flags; + + resp_req->dev.user_buf = (void *)MPIU_Malloc(type_size); + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* copy data to resp pkt header */ - void *src = fop_pkt->addr, *dest = fop_resp_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)fop_resp_pkt->immed_len); + void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; + mpi_errno = immed_copy(src, dest, type_size); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + MPIU_Memcpy(resp_req->dev.user_buf, fop_pkt->addr, + type_size); + } /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); int one = 1; - (*uop)(fop_pkt->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { + (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + } + else { + (*uop)(lock_entry->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + } } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,lock_entry->vc); mpi_errno = MPIDI_CH3_iStartMsg(lock_entry->vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); @@ -1179,9 +1316,24 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPID_Request_release(resp_req); } } + } + else { + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_resp_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = type_size; + iovcnt = 2; + + mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, resp_req, iov, iovcnt); + if (mpi_errno != MPI_SUCCESS) { + MPID_Request_release(resp_req); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + goto fn_exit; + } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, fop_pkt->type, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1201,7 +1353,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPI_Aint len; int mpi_errno = MPI_SUCCESS; - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -1220,7 +1372,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len); + MPIU_Memcpy((void *) &cas_resp_pkt->info.data, cas_pkt->addr, len); /* Compare and replace if equal */ if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) { @@ -1257,7 +1409,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, cas_pkt->type, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1297,6 +1449,7 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent /* LOCK+OP packet */ switch(lock_entry->pkt.type) { case (MPIDI_CH3_PKT_PUT): + case (MPIDI_CH3_PKT_PUT_IMMED): mpi_errno = perform_put_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; @@ -1305,18 +1458,21 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; case (MPIDI_CH3_PKT_ACCUMULATE): + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): mpi_errno = perform_acc_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; case (MPIDI_CH3_PKT_GET_ACCUM): + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): mpi_errno = perform_get_acc_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; case (MPIDI_CH3_PKT_FOP): + case (MPIDI_CH3_PKT_FOP_IMMED): mpi_errno = perform_fop_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; - case (MPIDI_CH3_PKT_CAS): + case (MPIDI_CH3_PKT_CAS_IMMED): mpi_errno = perform_cas_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c index a1e586b..84ad691 100644 --- a/src/mpid/ch3/src/ch3u_handle_send_req.c +++ b/src/mpid/ch3/src/ch3u_handle_send_req.c @@ -217,7 +217,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS_IMMED, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c index d9bdac0..910f297 100644 --- a/src/mpid/ch3/src/ch3u_request.c +++ b/src/mpid/ch3/src/ch3u_request.c @@ -93,7 +93,6 @@ MPID_Request * MPID_Request_create(void) req->dev.OnDataAvail = NULL; req->dev.OnFinal = NULL; req->dev.user_buf = NULL; - req->dev.final_user_buf = NULL; req->dev.drop_data = FALSE; #ifdef MPIDI_CH3_REQUEST_INIT MPIDI_CH3_REQUEST_INIT(req); diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c index 92b4961..3d97174 100644 --- a/src/mpid/ch3/src/ch3u_rma_oplist.c +++ b/src/mpid/ch3/src/ch3u_rma_oplist.c @@ -366,7 +366,9 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe win_ptr->active_req_cnt++; if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending PUT/ACC operation. */ } @@ -401,7 +403,9 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe &(target->dt_op_list_tail), curr_op); } else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list), &(target->write_op_list_tail), curr_op); } diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 29a65dc..a43a606 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -121,6 +121,9 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_put_t *put_pkt = NULL; + MPI_Aint origin_type_size; + size_t immed_len, len; + int use_immed_pkt = FALSE; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -155,30 +158,48 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); + + /* Judge if we can use IMMED data packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size, + size_t); + if (len <= immed_len) + use_immed_pkt = TRUE; + } + /* Judge if this operation is an piggyback candidate */ if (!new_ptr->is_dt) { - size_t len; - MPI_Aint origin_type_size; - MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t); - if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES, - MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) + if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) new_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ put_pkt = &(new_ptr->pkt.put); - MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + + if (use_immed_pkt) { + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT_IMMED); + } + else { + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + } + put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; put_pkt->count = target_count; put_pkt->datatype = target_datatype; - put_pkt->dataloop_size = 0; + put_pkt->info.dataloop_size = 0; put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; put_pkt->source_win_handle = win_ptr->handle; - put_pkt->immed_len = 0; put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(put_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -288,6 +309,9 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; + MPI_Aint target_type_size; + size_t immed_len, len; + int use_immed_resp_pkt = FALSE; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -322,6 +346,18 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(target_datatype, target_type_size); + MPIU_Assign_trunc(len, target_count * target_type_size, size_t); + + /* Judge if we can use IMMED data response packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (len <= immed_len) + use_immed_resp_pkt = TRUE; + } + /* Judge if this operation is an piggyback candidate. */ if (!new_ptr->is_dt) { new_ptr->piggyback_lock_candidate = 1; @@ -335,10 +371,12 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype win_ptr->disp_units[target_rank] * target_disp; get_pkt->count = target_count; get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; + get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_resp_pkt) + get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -450,6 +488,9 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; + MPI_Aint origin_type_size; + size_t immed_len, len; + int use_immed_pkt = FALSE; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -483,14 +524,21 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); + + /* Judge if we can use IMMED data packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size, + size_t); + if (len <= immed_len) + use_immed_pkt = TRUE; + } + /* Judge if this operation is an piggyback candidate. */ if (!new_ptr->is_dt) { - size_t len; - MPI_Aint origin_type_size; - MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t); - if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES, - MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) + if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) new_ptr->piggyback_lock_candidate = 1; } @@ -498,17 +546,27 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp accum_pkt = &(new_ptr->pkt.accum); - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + if (use_immed_pkt) { + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE_IMMED); + } + else { + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + } + accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; accum_pkt->count = target_count; accum_pkt->datatype = target_datatype; - accum_pkt->dataloop_size = 0; + accum_pkt->info.dataloop_size = 0; accum_pkt->op = op; accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; accum_pkt->source_win_handle = win_ptr->handle; - accum_pkt->immed_len = 0; accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(accum_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -634,6 +692,9 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (op == MPI_NO_OP) { /* Convert GAcc to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; + MPI_Aint target_type_size; + size_t len, immed_len; + int use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -659,6 +720,18 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(target_datatype, target_type_size); + MPIU_Assign_trunc(len, target_count * target_type_size, size_t); + + /* Judge if we can use IMMED data response packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (len <= immed_len) + use_immed_resp_pkt = TRUE; + } + /* Judge if this operation is a piggyback candidate */ if (!new_ptr->is_dt) { new_ptr->piggyback_lock_candidate = 1; @@ -672,14 +745,19 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, win_ptr->disp_units[target_rank] * target_disp; get_pkt->count = target_count; get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; + get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_resp_pkt == TRUE) + get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } else { MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; + MPI_Aint origin_type_size, target_type_size; + size_t immed_len, orig_len, tar_len; + int use_immed_pkt = FALSE, use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -715,31 +793,60 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(orig_len, origin_count * origin_type_size, size_t); + + MPID_Datatype_get_size_macro(target_datatype, target_type_size); + MPIU_Assign_trunc(tar_len, target_count * target_type_size, size_t); + + /* Judge if we can use IMMED data packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size, + size_t); + if (orig_len <= immed_len) + use_immed_pkt = TRUE; + + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (tar_len <= immed_len) + use_immed_resp_pkt = TRUE; + } + /* Judge if this operation is a piggyback candidate */ if (!new_ptr->is_dt) { - size_t len; - MPI_Aint origin_type_size; - MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t); - if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES, - MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) + if (orig_len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) new_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ get_accum_pkt = &(new_ptr->pkt.get_accum); - MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + + if (use_immed_pkt) { + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED); + } + else { + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + } + get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; get_accum_pkt->count = target_count; get_accum_pkt->datatype = target_datatype; - get_accum_pkt->dataloop_size = 0; + get_accum_pkt->info.dataloop_size = 0; get_accum_pkt->op = op; get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_accum_pkt->source_win_handle = win_ptr->handle; - get_accum_pkt->immed_len = 0; get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(get_accum_pkt->info.data); + mpi_errno = immed_copy(src, dest, orig_len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + if (use_immed_resp_pkt) + get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -953,6 +1060,8 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL; + MPI_Aint type_size; + void *src = NULL, *dest = NULL; /* Append this operation to the RMA ops queue */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -976,7 +1085,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, /************** Setting packet struct areas in operation ****************/ cas_pkt = &(new_ptr->pkt.cas); - MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS_IMMED); cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; cas_pkt->datatype = datatype; @@ -984,6 +1093,19 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, cas_pkt->source_win_handle = win_ptr->handle; cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /* REQUIRE: All datatype arguments must be of the same, builtin + * type and counts must be 1. */ + MPID_Datatype_get_size_macro(datatype, type_size); + MPIU_Assert(type_size <= sizeof(MPIDI_CH3_CAS_Immed_u)); + + src = (void *)origin_addr, dest = (void *)(&(cas_pkt->origin_data)); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + src = (void *)compare_addr, dest = (void *)(&(cas_pkt->compare_data)); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); @@ -1090,6 +1212,9 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, if (op == MPI_NO_OP) { /* Convert FOP to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; + MPI_Aint target_type_size; + size_t immed_len; + int use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -1102,19 +1227,34 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /************** Setting packet struct areas in operation ****************/ + MPID_Datatype_get_size_macro(datatype, target_type_size); + MPIU_Assert(target_type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); + + /* Judege if we can use IMMED data for response packet */ + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (target_type_size <= immed_len) + use_immed_resp_pkt = TRUE; + get_pkt = &(new_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; get_pkt->count = 1; get_pkt->datatype = datatype; - get_pkt->dataloop_size = 0; + get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_resp_pkt == TRUE) + get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } else { MPIDI_CH3_Pkt_fop_t *fop_pkt; + MPI_Aint type_size; + size_t immed_len; + int use_immed_pkt = FALSE, use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -1129,17 +1269,40 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /************** Setting packet struct areas in operation ****************/ + MPID_Datatype_get_size_macro(datatype, type_size); + MPIU_Assert(type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); + + /* Judge if we can use IMMED data packet */ + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/type_size)*type_size, + size_t); + if (type_size <= immed_len) { + use_immed_pkt = TRUE; + use_immed_resp_pkt = TRUE; + } + fop_pkt = &(new_ptr->pkt.fop); - MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + if (use_immed_pkt) { + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP_IMMED); + } + else { + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + } fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; fop_pkt->datatype = datatype; fop_pkt->op = op; fop_pkt->source_win_handle = win_ptr->handle; fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - fop_pkt->immed_len = 0; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(fop_pkt->info.data); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + if (use_immed_resp_pkt) + fop_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index baa19a6..4163baa 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -214,6 +214,27 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } + MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); + + if (pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { + + /* Immed packet type is used when target datatype is predefined datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)); + + /* copy data from packet header to target buffer */ + MPIU_Memcpy(put_pkt->addr, put_pkt->info.data, put_pkt->count*type_size); + + /* trigger final action */ + mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + put_pkt->flags, put_pkt->source_win_handle); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_PUT); + /* get start location of data and length of data */ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); @@ -232,28 +253,8 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); req->dev.datatype = put_pkt->datatype; - MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); req->dev.recv_data_sz = type_size * put_pkt->count; - - if (put_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, put_pkt->data, (size_t)put_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + put_pkt->immed_len); - req->dev.recv_data_sz -= put_pkt->immed_len; - } - - if (req->dev.recv_data_sz == 0) { - /* All data received, trigger req handler. */ - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - mpi_errno = MPIDI_CH3_ReqHandler_PutRecvComplete(vc, req, &complete); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } + MPIU_Assert(req->dev.recv_data_sz > 0); mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -284,23 +285,23 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(put_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - put_pkt->dataloop_size); + put_pkt->info.dataloop_size); } /* if we received all of the dtype_info and dataloop, copy it * now and call the handler, otherwise set the iov and let the * channel copy it */ - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - put_pkt->dataloop_size); + put_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_PutDerivedDTRecvComplete(vc, req, &complete); @@ -315,7 +316,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) req->dev.dtype_info); req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = put_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = put_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -331,6 +332,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); } + } fn_exit: @@ -351,7 +353,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_CH3_Pkt_get_t *get_pkt = &pkt->get; MPID_Request *req = NULL; MPID_IOV iov[MPID_IOV_LIMIT]; - int complete; + int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; MPID_Win *win_ptr; @@ -391,6 +393,10 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)); + } + if (MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)) { /* basic datatype. send the data. */ MPIDI_CH3_Pkt_t upkt; @@ -403,7 +409,12 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendComplete; req->kind = MPID_REQUEST_SEND; - MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP_IMMED); + } + else { + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + } get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -414,31 +425,16 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_resp_pkt->source_win_handle = get_pkt->source_win_handle; - get_resp_pkt->immed_len = 0; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); - MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t); - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_resp_pkt->immed_len > 0) { - void *src = get_pkt->addr; - void *dest = (void*) get_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } + MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t); + void *src = (void*)(get_pkt->addr), *dest = (void*)(get_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (len == (size_t)get_resp_pkt->immed_len) { - /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); iovcnt = 1; @@ -446,8 +442,8 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)get_pkt->addr + get_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = get_pkt->count * type_size - get_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)get_pkt->addr); + iov[1].MPID_IOV_LEN = get_pkt->count * type_size; iovcnt = 2; } @@ -483,23 +479,23 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(get_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(get_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - get_pkt->dataloop_size); + get_pkt->info.dataloop_size); } /* if we received all of the dtype_info and dataloop, copy it * now and call the handler, otherwise set the iov and let the * channel copy it */ - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - get_pkt->dataloop_size); + get_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete(vc, req, &complete); @@ -512,7 +508,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = get_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = get_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -566,6 +562,31 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } + if (pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { + /* Immed packet type is used when target datatype is predefined datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)); + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + mpi_errno = do_accumulate_op(accum_pkt->info.data, accum_pkt->addr, + accum_pkt->count, accum_pkt->datatype, accum_pkt->op); + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + /* trigger final action */ + mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + accum_pkt->flags, accum_pkt->source_win_handle); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_ACCUMULATE); + req = MPID_Request_create(); MPIU_Object_set_ref(req, 1); *rreqp = req; @@ -601,28 +622,10 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } req->dev.user_buf = tmp_buf; - req->dev.final_user_buf = req->dev.user_buf; MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size); req->dev.recv_data_sz = type_size * accum_pkt->count; - - if (accum_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, accum_pkt->data, (size_t)accum_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + accum_pkt->immed_len); - req->dev.recv_data_sz -= accum_pkt->immed_len; - } - - if (req->dev.recv_data_sz == 0) { - /* All data received, trigger req handler. */ - *buflen = sizeof(MPIDI_CH3_Pkt_t); - mpi_errno = MPIDI_CH3_ReqHandler_AccumRecvComplete(vc, req, &complete); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } + MPIU_Assert(req->dev.recv_data_sz > 0); mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -653,20 +656,20 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(accum_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - accum_pkt->dataloop_size); + accum_pkt->info.dataloop_size); } - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - accum_pkt->dataloop_size); + accum_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete(vc, req, &complete); @@ -681,12 +684,13 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = accum_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = accum_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); } } + } if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -741,6 +745,118 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } + MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); + + if (pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { + size_t len; + void *src = NULL, *dest = NULL; + MPID_Request *resp_req = NULL; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &upkt.get_accum_resp; + MPID_IOV iov[MPID_IOV_LIMIT]; + int iovcnt; + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + /* Immed packet type is used when target datatype is predefined datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)); + + resp_req = MPID_Request_create(); + resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; + resp_req->dev.source_win_handle = get_accum_pkt->source_win_handle; + resp_req->dev.flags = get_accum_pkt->flags; + + MPIDI_Request_set_type(resp_req, MPIDI_REQUEST_TYPE_GET_ACCUM_RESP); + resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumSendComplete; + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumSendComplete; + resp_req->kind = MPID_REQUEST_SEND; + if (!(get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { + tmp_buf = MPIU_Malloc(get_accum_pkt->count * type_size); + if (!tmp_buf) { + MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d", + get_accum_pkt->count * type_size); + } + resp_req->dev.user_buf = tmp_buf; + } + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + /* Calculate the length of reponse data, ensure that it fits into immed packet. */ + MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); + + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); + } + else { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); + } + + get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; + get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; + get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; + get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) + get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; + if ((get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || + (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) + get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { + /* copy data from target buffer to response packet header */ + src = (void *) (get_accum_pkt->addr), dest = (void *) (get_accum_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + MPIU_Memcpy(resp_req->dev.user_buf, get_accum_pkt->addr, + get_accum_pkt->count * type_size); + } + + /* perform accumulate operation. */ + mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, + get_accum_pkt->count, get_accum_pkt->datatype, + get_accum_pkt->op); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); + iovcnt = 1; + } + else { + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = get_accum_pkt->count * type_size; + iovcnt = 2; + } + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iSendv(vc, resp_req, iov, iovcnt); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPIU_Object_set_ref(resp_req, 0); + MPIDI_CH3_Request_destroy(resp_req); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + /* --END ERROR HANDLING-- */ + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_GET_ACCUM); + req = MPID_Request_create(); MPIU_Object_set_ref(req, 1); *rreqp = req; @@ -776,30 +892,9 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } req->dev.user_buf = tmp_buf; - req->dev.final_user_buf = req->dev.user_buf; - MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); req->dev.recv_data_sz = type_size * get_accum_pkt->count; - - if (get_accum_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, get_accum_pkt->data, (size_t)get_accum_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + get_accum_pkt->immed_len); - req->dev.recv_data_sz -= get_accum_pkt->immed_len; - } - - if (req->dev.recv_data_sz == 0) { - /* All data received. */ - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - mpi_errno = MPIDI_CH3_ReqHandler_GaccumRecvComplete(vc, req, &complete); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } + MPIU_Assert(req->dev.recv_data_sz > 0); mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -830,20 +925,20 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(get_accum_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(get_accum_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - get_accum_pkt->dataloop_size); + get_accum_pkt->info.dataloop_size); } - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - get_accum_pkt->dataloop_size); + get_accum_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_GaccumDerivedDTRecvComplete(vc, req, &complete); @@ -858,7 +953,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = get_accum_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = get_accum_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); } @@ -869,6 +964,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); } + } fn_exit: MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_accum); @@ -923,7 +1019,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); *rreqp = NULL; - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -942,7 +1038,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len); + MPIU_Memcpy((void *) &cas_resp_pkt->info.data, cas_pkt->addr, len); /* Compare and replace if equal */ if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) { @@ -978,7 +1074,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Request_release(req); } - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, vc, cas_pkt->type, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1034,7 +1130,7 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); MPID_Datatype_get_size_macro(req->dev.datatype, len); - MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->data, len); + MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->info.data, len); MPIDI_CH3U_Request_complete(req); *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -1064,6 +1160,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Request *rreq = NULL; int acquire_lock_fail = 0; MPID_Win *win_ptr = NULL; + MPI_Aint type_size; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); @@ -1088,7 +1185,13 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, (*buflen) = sizeof(MPIDI_CH3_Pkt_t); (*rreqp) = NULL; - MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); + MPID_Datatype_get_size_macro(fop_pkt->datatype, type_size); + + if (pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { + + MPIU_Assert(fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP); + + MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED); fop_resp_pkt->request_handle = fop_pkt->request_handle; fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -1099,21 +1202,20 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - fop_resp_pkt->immed_len = fop_pkt->immed_len; if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); /* copy data to resp pkt header */ - void *src = fop_pkt->addr, *dest = fop_resp_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)fop_resp_pkt->immed_len); + void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; + mpi_errno = immed_copy(src, dest, type_size); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); int one = 1; - (*uop)(fop_pkt->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); } if (win_ptr->shm_allocated == TRUE) @@ -1145,9 +1247,68 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } } - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, vc, fop_pkt->type, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_FOP); + + MPIU_Assert(!(fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + + MPID_Request *req = NULL; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + MPI_Aint extent; + int complete = 0; + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); + *rreqp = req; + + req->dev.op = fop_pkt->op; + req->dev.real_user_buf = fop_pkt->addr; + req->dev.target_win_handle = fop_pkt->target_win_handle; + req->dev.source_win_handle = fop_pkt->source_win_handle; + req->dev.flags = fop_pkt->flags; + req->dev.resp_request_handle = fop_pkt->request_handle; + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPRecvComplete; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_FOPRecvComplete; + req->dev.datatype = fop_pkt->datatype; + req->dev.user_count = 1; + + /* get start location of data and length of data */ + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + MPID_Datatype_get_extent_macro(fop_pkt->datatype, extent); + + req->dev.user_buf = MPIU_Malloc(extent); + if (!req->dev.user_buf) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + extent); + } + + req->dev.recv_data_sz = type_size; + MPIU_Assert(req->dev.recv_data_sz > 0); + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + if (complete) { + mpi_errno = MPIDI_CH3_ReqHandler_FOPRecvComplete(vc, req, &complete); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + if (complete) { + *rreqp = NULL; + goto fn_exit; + } + } + } fn_exit: MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop); @@ -1172,6 +1333,10 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &pkt->fop_resp; MPID_Request *req = NULL; MPID_Win *win_ptr = NULL; + MPI_Aint type_size; + MPIDI_msg_sz_t data_len; + char *data_buf = NULL; + int complete = 0; int target_rank = fop_resp_pkt->target_rank; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); @@ -1183,9 +1348,7 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPID_Win_get_ptr(fop_resp_pkt->source_win_handle, win_ptr); - /* Copy data to result buffer on orgin */ MPID_Request_get_ptr(fop_resp_pkt->request_handle, req); - MPIU_Memcpy(req->dev.user_buf, fop_resp_pkt->data, (size_t)fop_resp_pkt->immed_len); /* decrement ack_counter */ if (fop_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1202,9 +1365,37 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), if (mpi_errno) MPIU_ERR_POP(mpi_errno); } + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + MPID_Datatype_get_size_macro(req->dev.datatype, type_size); + req->dev.recv_data_sz = type_size; + req->dev.user_count = 1; + + *rreqp = req; + + if (fop_resp_pkt->type == MPIDI_CH3_PKT_FOP_RESP_IMMED) { + MPIU_Memcpy(req->dev.user_buf, fop_resp_pkt->info.data, req->dev.recv_data_sz); + + /* return the number of bytes processed in this function */ + *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; + } + else { + MPIU_Assert(fop_resp_pkt->type == MPIDI_CH3_PKT_FOP_RESP); + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", + "MPIDI_CH3_PKT_FOP_RESP"); + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + } + + if (complete) { MPIDI_CH3U_Request_complete(req); - *buflen = sizeof(MPIDI_CH3_Pkt_t); *rreqp = NULL; + } fn_exit: MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop_resp); @@ -1226,7 +1417,7 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &pkt->get_accum_resp; MPID_Request *req; - int complete; + int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; int mpi_errno = MPI_SUCCESS; @@ -1266,20 +1457,17 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; - if (get_accum_resp_pkt->immed_len > 0) { - /* first copy IMMED data from pkt header to origin buffer */ - MPIU_Memcpy(req->dev.user_buf, get_accum_resp_pkt->data, (size_t)get_accum_resp_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + get_accum_resp_pkt->immed_len); - req->dev.recv_data_sz -= get_accum_resp_pkt->immed_len; - if (req->dev.recv_data_sz == 0) - complete = 1; + *rreqp = req; + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { + MPIU_Memcpy(req->dev.user_buf, get_accum_resp_pkt->info.data, req->dev.recv_data_sz); /* return the number of bytes processed in this function */ *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP); - if(req->dev.recv_data_sz > 0) { - *rreqp = req; mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_ACCUM_RESP"); @@ -1371,7 +1559,7 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), { MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &pkt->get_resp; MPID_Request *req; - int complete; + int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; int mpi_errno = MPI_SUCCESS; @@ -1411,20 +1599,18 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; - if (get_resp_pkt->immed_len > 0) { - /* first copy IMMED data from pkt header to origin buffer */ - MPIU_Memcpy(req->dev.user_buf, get_resp_pkt->data, (size_t)get_resp_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + get_resp_pkt->immed_len); - req->dev.recv_data_sz -= get_resp_pkt->immed_len; - if (req->dev.recv_data_sz == 0) - complete = 1; + *rreqp = req; + + if (get_resp_pkt->type == MPIDI_CH3_PKT_GET_RESP_IMMED) { + MPIU_Memcpy(req->dev.user_buf, get_resp_pkt->info.data, req->dev.recv_data_sz); /* return the number of bytes processed in this function */ *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; } + else { + MPIU_Assert(get_resp_pkt->type == MPIDI_CH3_PKT_GET_RESP); - if (req->dev.recv_data_sz > 0) { - *rreqp = req; mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_RESP"); @@ -1705,7 +1891,7 @@ int MPIDI_CH3_PktPrint_Put(FILE * fp, MPIDI_CH3_Pkt_t * pkt) MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->put.addr)); MPIU_DBG_PRINTF((" count ........ %d\n", pkt->put.count)); MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->put.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.dataloop_size)); + MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.info.dataloop_size)); MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->put.target_win_handle)); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->put.source_win_handle)); /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->put.win_ptr)); */ @@ -1718,7 +1904,7 @@ int MPIDI_CH3_PktPrint_Get(FILE * fp, MPIDI_CH3_Pkt_t * pkt) MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->get.addr)); MPIU_DBG_PRINTF((" count ........ %d\n", pkt->get.count)); MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->get.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.dataloop_size)); + MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.info.dataloop_size)); MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request_handle)); MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->get.target_win_handle)); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->get.source_win_handle)); @@ -1743,7 +1929,7 @@ int MPIDI_CH3_PktPrint_Accumulate(FILE * fp, MPIDI_CH3_Pkt_t * pkt) MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum.addr)); MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum.count)); MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.dataloop_size)); + MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.info.dataloop_size)); MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum.op)); MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum.target_win_handle)); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum.source_win_handle)); http://git.mpich.org/mpich.git/commitdiff/3a017faae893f03d2ad2fb4d24f037d964... commit 3a017faae893f03d2ad2fb4d24f037d964782e23 Author: Xin Zhao <[email protected]> Date: Fri Jan 23 10:28:46 2015 -0600 Code-refactoring for RMA operations routines. This patch just does code refactoring for RMA operation rountines to make the code structure clearer. This patch does not change any functionality. After code refactoring, in each operation routine, for non-SHM operations we do the work in the following order: (1) allocate a new op struct; (2) fill areas in op struct, except for packet struct in op struct; (3) initialize packet struct in op struct, fill areas in packet struct; (4) enqueue op to data structure on window. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 45df32b..29a65dc 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -128,17 +128,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - put_pkt = &(new_ptr->pkt.put); - MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); - put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - put_pkt->count = target_count; - put_pkt->datatype = target_datatype; - put_pkt->dataloop_size = 0; - put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - put_pkt->source_win_handle = win_ptr->handle; - put_pkt->immed_len = 0; - put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->origin_addr = (void *) origin_addr; @@ -152,14 +142,6 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->ureq = ureq; } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -184,8 +166,26 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->piggyback_lock_candidate = 1; } + /************** Setting packet struct areas in operation ****************/ + + put_pkt = &(new_ptr->pkt.put); + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + put_pkt->count = target_count; + put_pkt->datatype = target_datatype; + put_pkt->dataloop_size = 0; + put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + put_pkt->source_win_handle = win_ptr->handle; + put_pkt->immed_len = 0; + put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -295,16 +295,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - get_pkt = &(new_ptr->pkt.get); - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - get_pkt->count = target_count; - get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; - get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; - get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->origin_addr = origin_addr; @@ -318,14 +309,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->ureq = ureq; } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -344,8 +327,25 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->piggyback_lock_candidate = 1; } + /************** Setting packet struct areas in operation ****************/ + + get_pkt = &(new_ptr->pkt.get); + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_pkt->count = target_count; + get_pkt->datatype = target_datatype; + get_pkt->dataloop_size = 0; + get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_pkt->source_win_handle = win_ptr->handle; + get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -457,19 +457,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - accum_pkt = &(new_ptr->pkt.accum); - - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); - accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - accum_pkt->count = target_count; - accum_pkt->datatype = target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = op; - accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - accum_pkt->source_win_handle = win_ptr->handle; - accum_pkt->immed_len = 0; - accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = origin_count; @@ -482,14 +470,6 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp new_ptr->ureq = ureq; } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -514,8 +494,28 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp new_ptr->piggyback_lock_candidate = 1; } + /************** Setting packet struct areas in operation ****************/ + + accum_pkt = &(new_ptr->pkt.accum); + + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + accum_pkt->count = target_count; + accum_pkt->datatype = target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = op; + accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + accum_pkt->source_win_handle = win_ptr->handle; + accum_pkt->immed_len = 0; + accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -633,21 +633,20 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (op == MPI_NO_OP) { /* Convert GAcc to a Get */ - MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get); - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - get_pkt->count = target_count; - get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; - get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; - get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIDI_CH3_Pkt_get_t *get_pkt; + + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = result_addr; new_ptr->origin_count = result_count; new_ptr->origin_datatype = result_datatype; new_ptr->target_rank = target_rank; + new_ptr->ureq = NULL; /* reset user request */ + + /* Remember user request */ + if (ureq) { + new_ptr->ureq = ureq; + } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, dtp); @@ -664,21 +663,25 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (!new_ptr->is_dt) { new_ptr->piggyback_lock_candidate = 1; } + + /************** Setting packet struct areas in operation ****************/ + + get_pkt = &(new_ptr->pkt.get); + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_pkt->count = target_count; + get_pkt->datatype = target_datatype; + get_pkt->dataloop_size = 0; + get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_pkt->source_win_handle = win_ptr->handle; + get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; } else { - MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt = &(new_ptr->pkt.get_accum); - MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); - get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - get_accum_pkt->count = target_count; - get_accum_pkt->datatype = target_datatype; - get_accum_pkt->dataloop_size = 0; - get_accum_pkt->op = op; - get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_accum_pkt->source_win_handle = win_ptr->handle; - get_accum_pkt->immed_len = 0; - get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; + + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = origin_count; @@ -687,6 +690,12 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->result_count = result_count; new_ptr->result_datatype = result_datatype; new_ptr->target_rank = target_rank; + new_ptr->ureq = NULL; /* reset user request */ + + /* Remember user request */ + if (ureq) { + new_ptr->ureq = ureq; + } /* if source or target datatypes are derived, increment their * reference counts */ @@ -716,13 +725,21 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) new_ptr->piggyback_lock_candidate = 1; } - } - new_ptr->ureq = NULL; /* reset user request */ + /************** Setting packet struct areas in operation ****************/ - /* Remember user request */ - if (ureq) { - new_ptr->ureq = ureq; + get_accum_pkt = &(new_ptr->pkt.get_accum); + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_accum_pkt->count = target_count; + get_accum_pkt->datatype = target_datatype; + get_accum_pkt->dataloop_size = 0; + get_accum_pkt->op = op; + get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_accum_pkt->source_win_handle = win_ptr->handle; + get_accum_pkt->immed_len = 0; + get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -943,14 +960,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - cas_pkt = &(new_ptr->pkt.cas); - MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); - cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - cas_pkt->datatype = datatype; - cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - cas_pkt->source_win_handle = win_ptr->handle; - cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; @@ -963,6 +973,17 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, new_ptr->piggyback_lock_candidate = 1; /* CAS is always able to piggyback LOCK */ new_ptr->ureq = NULL; /* reset user request */ + /************** Setting packet struct areas in operation ****************/ + + cas_pkt = &(new_ptr->pkt.cas); + MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + cas_pkt->datatype = datatype; + cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + cas_pkt->source_win_handle = win_ptr->handle; + cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); @@ -1068,7 +1089,20 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, if (op == MPI_NO_OP) { /* Convert FOP to a Get */ - MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get); + MPIDI_CH3_Pkt_get_t *get_pkt; + + /******************** Setting operation struct areas ***********************/ + + new_ptr->origin_addr = result_addr; + new_ptr->origin_count = 1; + new_ptr->origin_datatype = datatype; + new_ptr->target_rank = target_rank; + new_ptr->piggyback_lock_candidate = 1; + new_ptr->ureq = NULL; /* reset user request */ + + /************** Setting packet struct areas in operation ****************/ + + get_pkt = &(new_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; @@ -1078,15 +1112,24 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + } + else { + MPIDI_CH3_Pkt_fop_t *fop_pkt; - new_ptr->origin_addr = result_addr; + /******************** Setting operation struct areas ***********************/ + + new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; new_ptr->origin_datatype = datatype; + new_ptr->result_addr = result_addr; + new_ptr->result_datatype = datatype; new_ptr->target_rank = target_rank; new_ptr->piggyback_lock_candidate = 1; - } - else { - MPIDI_CH3_Pkt_fop_t *fop_pkt = &(new_ptr->pkt.fop); + new_ptr->ureq = NULL; /* reset user request */ + + /************** Setting packet struct areas in operation ****************/ + + fop_pkt = &(new_ptr->pkt.fop); MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + @@ -1097,18 +1140,8 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; fop_pkt->immed_len = 0; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = 1; - new_ptr->origin_datatype = datatype; - new_ptr->result_addr = result_addr; - new_ptr->result_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->piggyback_lock_candidate = 1; } - new_ptr->ureq = NULL; /* reset user request */ - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); http://git.mpich.org/mpich.git/commitdiff/81e2b2742b9a690e68d40d93e42a6d3684... commit 81e2b2742b9a690e68d40d93e42a6d3684206fa3 Author: Xin Zhao <[email protected]> Date: Mon Feb 9 17:04:22 2015 -0800 Remove lock_type and origin_rank areas from RMA packet. Originally we added lock_type and origin_rank areas in RMA packet, in order to piggyback passive lock request with RMA operations. However, those areas potentially enlarged the packet union size, and actually they are not necessary and can be completetly avoided. "Lock_type" is used to remember what types of lock (shared or exclusive) the origin wants to acquire on the target. To remove it from RMA packet, we use flags (already exists in RMA packet) to remember such information. "Origin_rank" is used to remember which origin has sent lock request to the target, so that when the lock is granted to this origin later, the target can send ack to that origin. Actually the target does not need to store origin_rank but can only store origin_vc, which is known from progress engine on target side. Therefore, we can completely remove origin_rank from RMA packet. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index c64a226..a443521 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -339,8 +339,6 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, rma_op->request = NULL; put_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - put_pkt->lock_type = target_ptr->lock_type; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); @@ -413,9 +411,6 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, accum_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - accum_pkt->lock_type = target_ptr->lock_type; - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); @@ -516,9 +511,6 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - get_accum_pkt->lock_type = target_ptr->lock_type; - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); @@ -650,9 +642,6 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - get_pkt->lock_type = target_ptr->lock_type; - comm_ptr = win_ptr->comm_ptr; MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); @@ -750,8 +739,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - cas_pkt->lock_type = target_ptr->lock_type; MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); @@ -821,8 +808,6 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, fop_pkt->request_handle = resp_req->handle; fop_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - fop_pkt->lock_type = target_ptr->lock_type; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); diff --git a/src/mpid/ch3/include/mpid_rma_lockqueue.h b/src/mpid/ch3/include/mpid_rma_lockqueue.h index 0e1919b..1c98ced 100644 --- a/src/mpid/ch3/include/mpid_rma_lockqueue.h +++ b/src/mpid/ch3/include/mpid_rma_lockqueue.h @@ -32,6 +32,7 @@ static inline MPIDI_RMA_Lock_entry_t *MPIDI_CH3I_Win_lock_entry_alloc(MPID_Win * if (new_ptr != NULL) { new_ptr->next = NULL; new_ptr->pkt = (*pkt); + new_ptr->vc = NULL; new_ptr->data = NULL; new_ptr->data_size = 0; new_ptr->all_data_recved = 0; diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 9089b1a..a1060d7 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -131,6 +131,7 @@ extern MPIDI_RMA_Win_list_t *MPIDI_RMA_Win_list, *MPIDI_RMA_Win_list_tail; typedef struct MPIDI_RMA_Lock_entry { struct MPIDI_RMA_Lock_entry *next; MPIDI_CH3_Pkt_t pkt; /* all information for this request packet */ + MPIDI_VC_t *vc; void *data; /* for queued PUTs / ACCs / GACCs, data is copied here */ int data_size; int all_data_recved; /* indicate if all data has been received */ diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 852160b..f6e5327 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -109,21 +109,20 @@ typedef enum { /* These flags can be "OR'ed" together */ typedef enum { MPIDI_CH3_PKT_FLAG_NONE = 0, - MPIDI_CH3_PKT_FLAG_RMA_LOCK = 1, - MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 2, - MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 4, - MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 8, - MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 16, - MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 32, - MPIDI_CH3_PKT_FLAG_RMA_SHARED = 64, - MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE = 128, - MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 256, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 512, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 1024, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 2048, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 4096, - MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 8192, - MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 16384 + MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2, + MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4, + MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8, + MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16, + MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32, + MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64, + MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 128, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048, + MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096, + MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192 } MPIDI_CH3_Pkt_flags_t; typedef struct MPIDI_CH3_Pkt_send { @@ -330,72 +329,6 @@ MPIDI_CH3_PKT_DEFS } \ } -#define MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(pkt_, lock_type_, err_) \ - { \ - /* This macro returns lock type in RMA operation \ - packets (PUT, GET, ACC, GACC, FOP, CAS) and RMA control \ - packets (LOCK). */ \ - err_ = MPI_SUCCESS; \ - switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - lock_type_ = (pkt_).put.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_GET): \ - lock_type_ = (pkt_).get.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - lock_type_ = (pkt_).accum.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - lock_type_ = (pkt_).get_accum.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_CAS): \ - lock_type_ = (pkt_).cas.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - lock_type_ = (pkt_).fop.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_LOCK): \ - lock_type_ = (pkt_).lock.lock_type; \ - break; \ - default: \ - MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ - } \ - } - -#define MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK(pkt_, origin_rank_, err_) \ - { \ - /* This macro returns origin rank (used in acquiring lock) in \ - RMA operation packets (PUT, GET, ACC, GACC, FOP, CAS) and \ - RMA control packets (LOCK). */ \ - err_ = MPI_SUCCESS; \ - switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - origin_rank_ = (pkt_).put.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_GET): \ - origin_rank_ = (pkt_).get.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - origin_rank_ = (pkt_).accum.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - origin_rank_ = (pkt_).get_accum.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_CAS): \ - origin_rank_ = (pkt_).cas.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - origin_rank_ = (pkt_).fop.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_LOCK): \ - origin_rank_ = (pkt_).lock.origin_rank; \ - break; \ - default: \ - MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ - } \ - } - #define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_) \ { \ /* This macro returns flags in RMA operation packets (PUT, GET, \ @@ -435,6 +368,9 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_CAS_RESP): \ flags_ = (pkt_).cas_resp.flags; \ break; \ + case (MPIDI_CH3_PKT_LOCK): \ + flags_ = (pkt_).lock.flags; \ + break; \ case (MPIDI_CH3_PKT_UNLOCK): \ flags_ = (pkt_).unlock.flags; \ break; \ @@ -488,6 +424,9 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_CAS_RESP): \ (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ + case (MPIDI_CH3_PKT_LOCK): \ + (pkt_).lock.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ case (MPIDI_CH3_PKT_UNLOCK): \ (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ @@ -646,8 +585,6 @@ typedef struct MPIDI_CH3_Pkt_put { * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_put_t; typedef struct MPIDI_CH3_Pkt_get { @@ -665,8 +602,6 @@ typedef struct MPIDI_CH3_Pkt_get { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_t; typedef struct MPIDI_CH3_Pkt_get_resp { @@ -698,8 +633,6 @@ typedef struct MPIDI_CH3_Pkt_accum { * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum { @@ -720,8 +653,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum { * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum_resp { @@ -749,8 +680,6 @@ typedef struct MPIDI_CH3_Pkt_cas { * in passive target rma. Otherwise set to NULL*/ MPIDI_CH3_CAS_Immed_u origin_data; MPIDI_CH3_CAS_Immed_u compare_data; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_cas_t; typedef struct MPIDI_CH3_Pkt_cas_resp { @@ -777,8 +706,6 @@ typedef struct MPIDI_CH3_Pkt_fop { * in passive target rma. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_fop_t; typedef struct MPIDI_CH3_Pkt_fop_resp { @@ -794,10 +721,9 @@ typedef struct MPIDI_CH3_Pkt_fop_resp { typedef struct MPIDI_CH3_Pkt_lock { MPIDI_CH3_Pkt_type_t type; - int lock_type; + MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; - int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 6d9594d..d7a720d 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -32,8 +32,13 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; lock_pkt->source_win_handle = win_ptr->handle; - lock_pkt->lock_type = lock_type; - lock_pkt->origin_rank = win_ptr->comm_ptr->rank; + lock_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (lock_type == MPI_LOCK_SHARED) + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; + else { + MPIU_Assert(lock_type == MPI_LOCK_EXCLUSIVE); + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE; + } MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_pkt, sizeof(*lock_pkt), &req); @@ -321,6 +326,7 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, new_ptr = MPIDI_CH3I_Win_lock_entry_alloc(win_ptr, pkt); if (new_ptr != NULL) { MPL_LL_APPEND(win_ptr->lock_queue, win_ptr->lock_queue_tail, new_ptr); + new_ptr->vc = vc; } else { lock_discarded = 1; @@ -386,18 +392,16 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t new_pkt; MPIDI_CH3_Pkt_lock_t *lock_pkt = &new_pkt.lock; MPI_Win target_win_handle; - int lock_type, origin_rank; + MPIDI_CH3_Pkt_flags_t flags; MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE((*pkt), target_win_handle, mpi_errno); MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK((*pkt), origin_rank, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE((*pkt), lock_type, mpi_errno); + MPIDI_CH3_PKT_RMA_GET_FLAGS((*pkt), flags, mpi_errno); MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = target_win_handle; lock_pkt->source_win_handle = source_win_handle; - lock_pkt->lock_type = lock_type; - lock_pkt->origin_rank = origin_rank; + lock_pkt->flags = flags; /* replace original pkt with lock pkt */ new_ptr->pkt = new_pkt; @@ -571,7 +575,8 @@ static inline int adjust_op_piggybacked_with_lock (MPID_Win *win_ptr, op = target->pending_op_list; if (op != NULL) MPIDI_CH3_PKT_RMA_GET_FLAGS(op->pkt, op_flags, mpi_errno); - if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { + if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED) { if (!op->request) { @@ -664,10 +669,16 @@ static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type) MPIDI_CH3_Pkt_t pkt; MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt.lock; MPIDI_RMA_Lock_entry_t *new_ptr = NULL; + MPIDI_VC_t *my_vc; MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); - lock_pkt->lock_type = lock_type; - lock_pkt->origin_rank = win_ptr->comm_ptr->rank; + lock_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (lock_type == MPI_LOCK_SHARED) + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; + else { + MPIU_Assert(lock_type == MPI_LOCK_EXCLUSIVE); + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE; + } new_ptr = MPIDI_CH3I_Win_lock_entry_alloc(win_ptr, &pkt); if (new_ptr == NULL) { @@ -677,6 +688,8 @@ static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type) goto fn_exit; } MPL_LL_APPEND(win_ptr->lock_queue, win_ptr->lock_queue_tail, new_ptr); + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &my_vc); + new_ptr->vc = my_vc; new_ptr->all_data_recved = 1; } @@ -834,9 +847,16 @@ static inline int check_piggyback_lock(MPID_Win *win_ptr, MPIDI_VC_t *vc, (*reqp) = NULL; MPIDI_CH3_PKT_RMA_GET_FLAGS((*pkt), flags, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE((*pkt), lock_type, mpi_errno); + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { + + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) + lock_type = MPI_LOCK_SHARED; + else { + MPIU_Assert(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + lock_type = MPI_LOCK_EXCLUSIVE; + } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { /* cannot acquire the lock, queue up this operation. */ mpi_errno = enqueue_lock_origin(win_ptr, vc, pkt, buflen, reqp); @@ -859,7 +879,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_ACCUMULATE) { /* This is PUT or ACC */ - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { MPIDI_CH3_Pkt_flags_t pkt_flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -871,7 +892,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Progress_signal_completion(); } if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) { - if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)) { + if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE)) { /* If op is piggybacked with both LOCK and FLUSH, we only send LOCK ACK back, do not send FLUSH ACK. */ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, @@ -888,7 +910,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Progress_signal_completion(); } if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { - if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)) { + if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE)) { /* If op is piggybacked with both LOCK and UNLOCK, we only send LOCK ACK back, do not send FLUSH (UNLOCK) ACK. */ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 79dc318..302e167 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -238,7 +238,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_accum_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -595,7 +596,8 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -831,7 +833,6 @@ static int create_derived_datatype(MPID_Request *req, MPID_Datatype **dtp) static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_entry_t *lock_entry) { MPIDI_CH3_Pkt_put_t *put_pkt = &((lock_entry->pkt).put); - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; if (lock_entry->data == NULL) { @@ -846,11 +847,8 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, put_pkt->origin_rank, &vc); - /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_PUT, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_PUT, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -866,7 +864,6 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp; MPIDI_CH3_Pkt_get_t *get_pkt = &((lock_entry->pkt).get); MPID_Request *sreq = NULL; - MPIDI_VC_t *vc = NULL; MPI_Aint type_size; size_t len; int iovcnt; @@ -894,7 +891,8 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -938,10 +936,7 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en iovcnt = 2; } - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, get_pkt->origin_rank, &vc); - - mpi_errno = MPIDI_CH3_iSendv(vc, sreq, iov, iovcnt); + mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt); if (mpi_errno != MPI_SUCCESS) { MPID_Request_release(sreq); MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -957,7 +952,6 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_entry_t *lock_entry) { MPIDI_CH3_Pkt_accum_t *acc_pkt = &((lock_entry->pkt).accum); - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; MPIU_Assert(lock_entry->all_data_recved == 1); @@ -980,10 +974,7 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, acc_pkt->origin_rank, &vc); - - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_ACCUMULATE, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_ACCUMULATE, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1000,7 +991,6 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &upkt.get_accum_resp; MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt = &((lock_entry->pkt).get_accum); MPID_Request *sreq = NULL; - MPIDI_VC_t *vc = NULL; MPI_Aint type_size; size_t len; int iovcnt; @@ -1065,7 +1055,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1108,10 +1099,7 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc iovcnt = 2; } - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, get_accum_pkt->origin_rank, &vc); - - mpi_errno = MPIDI_CH3_iSendv(vc, sreq, iov, iovcnt); + mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt); if (mpi_errno != MPI_SUCCESS) { MPID_Request_release(sreq); MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -1130,21 +1118,18 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; MPIDI_CH3_Pkt_fop_t *fop_pkt = &((lock_entry->pkt).fop); MPID_Request *resp_req = NULL; - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; /* FIXME: this function is same with PktHandler_FOP(), should do code refactoring on both of them. */ - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, fop_pkt->origin_rank, &vc); - MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); fop_resp_pkt->request_handle = fop_pkt->request_handle; fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1170,9 +1155,9 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); /* send back the original data */ - MPIU_THREAD_CS_ENTER(CH3COMM,vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); - MPIU_THREAD_CS_EXIT(CH3COMM,vc); + MPIU_THREAD_CS_ENTER(CH3COMM,lock_entry->vc); + mpi_errno = MPIDI_CH3_iStartMsg(lock_entry->vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); + MPIU_THREAD_CS_EXIT(CH3COMM,lock_entry->vc); MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); if (resp_req != NULL) { @@ -1196,7 +1181,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_FOP, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1213,19 +1198,16 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &upkt.cas_resp; MPIDI_CH3_Pkt_cas_t *cas_pkt = &((lock_entry->pkt).cas); MPID_Request *send_req = NULL; - MPIDI_VC_t *vc = NULL; MPI_Aint len; int mpi_errno = MPI_SUCCESS; - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, cas_pkt->origin_rank, &vc); - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) cas_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1249,9 +1231,9 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); /* Send the response packet */ - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &send_req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_THREAD_CS_ENTER(CH3COMM, lock_entry->vc); + mpi_errno = MPIDI_CH3_iStartMsg(lock_entry->vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &send_req); + MPIU_THREAD_CS_EXIT(CH3COMM, lock_entry->vc); MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -1275,7 +1257,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_CAS, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1295,16 +1277,17 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent /* single LOCK request */ MPIDI_CH3_Pkt_lock_t *lock_pkt = &(lock_entry->pkt.lock); - if (lock_pkt->origin_rank == win_ptr->comm_ptr->rank) { - mpi_errno = handle_lock_ack(win_ptr, lock_pkt->origin_rank, + MPIDI_VC_t *my_vc = NULL; + + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &my_vc); + + if (lock_entry->vc == my_vc) { + mpi_errno = handle_lock_ack(win_ptr, win_ptr->comm_ptr->rank, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } else { - MPIDI_VC_t *vc = NULL; - MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, - lock_pkt->origin_rank, &vc); - mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, + mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(lock_entry->vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, lock_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1410,7 +1393,14 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) lock_entry_next = lock_entry->next; if (lock_entry->all_data_recved) { - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(lock_entry->pkt, requested_lock, mpi_errno); + MPIDI_CH3_Pkt_flags_t flags; + MPIDI_CH3_PKT_RMA_GET_FLAGS(lock_entry->pkt, flags, mpi_errno); + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) + requested_lock = MPI_LOCK_SHARED; + else { + MPIU_Assert(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + requested_lock = MPI_LOCK_EXCLUSIVE; + } if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, requested_lock) == 1) { /* dequeue entry from lock queue */ MPL_LL_DELETE(win_ptr->lock_queue, win_ptr->lock_queue_tail, lock_entry); @@ -1456,6 +1446,7 @@ int MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete( MPIDI_VC_t *vc, int requested_lock; MPI_Win target_win_handle; MPID_Win *win_ptr = NULL; + MPIDI_CH3_Pkt_flags_t flags; MPIDI_RMA_Lock_entry_t *lock_queue_entry = rreq->dev.lock_queue_entry; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_PIGGYBACKLOCKOPRECVCOMPLETE); @@ -1473,10 +1464,18 @@ int MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete( MPIDI_VC_t *vc, lock_queue_entry->all_data_recved = 1; /* try to acquire the lock here */ - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(lock_queue_entry->pkt, requested_lock, mpi_errno); + MPIDI_CH3_PKT_RMA_GET_FLAGS(lock_queue_entry->pkt, flags, mpi_errno); MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(lock_queue_entry->pkt, target_win_handle, mpi_errno); MPID_Win_get_ptr(target_win_handle, win_ptr); + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) { + requested_lock = MPI_LOCK_SHARED; + } + else { + MPIU_Assert(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + requested_lock = MPI_LOCK_EXCLUSIVE; + } + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, requested_lock) == 1) { /* dequeue entry from lock queue */ MPL_LL_DELETE(win_ptr->lock_queue, win_ptr->lock_queue_tail, lock_queue_entry); diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c index 4261903..92b4961 100644 --- a/src/mpid/ch3/src/ch3u_rma_oplist.c +++ b/src/mpid/ch3/src/ch3u_rma_oplist.c @@ -328,7 +328,12 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe /* piggyback on first OP. */ if (target->access_state == MPIDI_RMA_LOCK_CALLED) { MPIU_Assert(curr_op->piggyback_lock_candidate); - flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK; + if (target->lock_type == MPI_LOCK_SHARED) + flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; + else { + MPIU_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE); + flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE; + } target->access_state = MPIDI_RMA_LOCK_ISSUED; } first_op = 0; @@ -366,7 +371,8 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe PUT/ACC operation. */ } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { /* If this operation is piggybacked with LOCK, do not move it out of pending list, and do not complete the user request, because we diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index f05d110..45df32b 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -138,7 +138,6 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; put_pkt->source_win_handle = win_ptr->handle; put_pkt->immed_len = 0; - put_pkt->origin_rank = rank; put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; /* FIXME: For contig and very short operations, use a streamlined op */ @@ -305,7 +304,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype get_pkt->dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; - get_pkt->origin_rank = rank; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; /* FIXME: For contig and very short operations, use a streamlined op */ @@ -471,7 +469,6 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; accum_pkt->source_win_handle = win_ptr->handle; accum_pkt->immed_len = 0; - accum_pkt->origin_rank = rank; accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; @@ -645,7 +642,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_pkt->dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; - get_pkt->origin_rank = rank; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = result_addr; @@ -682,7 +678,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_accum_pkt->source_win_handle = win_ptr->handle; get_accum_pkt->immed_len = 0; - get_accum_pkt->origin_rank = rank; get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; @@ -955,7 +950,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, cas_pkt->datatype = datatype; cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; cas_pkt->source_win_handle = win_ptr->handle; - cas_pkt->origin_rank = rank; cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; @@ -1083,7 +1077,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, get_pkt->dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; - get_pkt->origin_rank = rank; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = result_addr; @@ -1103,7 +1096,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, fop_pkt->source_win_handle = win_ptr->handle; fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; fop_pkt->immed_len = 0; - fop_pkt->origin_rank = rank; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index a64e97c..baa19a6 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -406,7 +406,8 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -927,7 +928,8 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) cas_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1091,7 +1093,8 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1315,6 +1318,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt->lock; MPID_Win *win_ptr = NULL; + int lock_type; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); @@ -1328,7 +1332,14 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr); - if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_pkt->lock_type) == 1) { + if (lock_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) + lock_type = MPI_LOCK_SHARED; + else { + MPIU_Assert(lock_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + lock_type = MPI_LOCK_EXCLUSIVE; + } + + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 1) { /* send lock granted packet. */ mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, lock_pkt->source_win_handle); http://git.mpich.org/mpich.git/commitdiff/d46b848a7e574a92521abe02bb63ba9275... commit d46b848a7e574a92521abe02bb63ba9275512d14 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 12:04:46 2015 -0800 Add comments about RMA packet wrappers. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 8a062f4..852160b 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -194,6 +194,8 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_) \ { \ + /* This macro returns target_datatype in RMA operation \ + packets. (PUT, GET, ACC, GACC, CAS, FOP) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -221,6 +223,8 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_) \ { \ + /* This macro returns target_count in RMA operation \ + packets. (PUT, GET, ACC, GACC, CAS, FOP) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -246,6 +250,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_IMMED_LEN(pkt_, immed_len_, err_) \ { \ + /* This macro returns immed_len in RMA operation \ + packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ + packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -284,6 +291,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_) \ { \ + /* This macro returns pointer to immed data in RMA operation \ + packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ + packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -322,6 +332,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(pkt_, lock_type_, err_) \ { \ + /* This macro returns lock type in RMA operation \ + packets (PUT, GET, ACC, GACC, FOP, CAS) and RMA control \ + packets (LOCK). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -352,6 +365,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK(pkt_, origin_rank_, err_) \ { \ + /* This macro returns origin rank (used in acquiring lock) in \ + RMA operation packets (PUT, GET, ACC, GACC, FOP, CAS) and \ + RMA control packets (LOCK). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -382,6 +398,11 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_) \ { \ + /* This macro returns flags in RMA operation packets (PUT, GET, \ + ACC, GACC, FOP, CAS), RMA operation response packets \ + (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control \ + packets (UNLOCK) and RMA control response packets (LOCK_ACK, \ + LOCK_OP_ACK) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -430,6 +451,11 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_) \ { \ + /* This macro erases flags in RMA operation packets (PUT, GET, \ + ACC, GACC, FOP, CAS), RMA operation response packets \ + (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control \ + packets (UNLOCK) and RMA control response packets (LOCK_ACK, \ + LOCK_OP_ACK) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -478,6 +504,11 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_) \ { \ + /* This macro returns source_win_handle in RMA operation \ + packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation \ + response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP), \ + RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control \ + response packets (LOCK_ACK, LOCK_OP_ACK, FLUSH_ACK). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -535,6 +566,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_) \ { \ + /* This macro returns target_win_handle in RMA operation \ + packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control \ + packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -574,6 +608,8 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \ { \ + /* This macro sets dataloop_size in RMA operation packets \ + (PUT, GET, ACC, GACC) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ http://git.mpich.org/mpich.git/commitdiff/064e60ce4e6903c56c27db5d6d8790f4be... commit 064e60ce4e6903c56c27db5d6d8790f4be05f491 Author: Xin Zhao <[email protected]> Date: Tue Feb 10 12:04:25 2015 -0800 Modify packet wrappers to make them complete. Some packet wrappers did not include all packet types, this patch adds missed packet types to those wrappers. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index a3e1793..8a062f4 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -261,7 +261,21 @@ MPIDI_CH3_PKT_DEFS immed_len_ = (pkt_).fop.immed_len; \ break; \ case (MPIDI_CH3_PKT_CAS): \ - /* FIXME: we should deal with CAS here */ \ + /* Note that here we return size of origin data, not \ + size of compare data. */ \ + immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ + break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + immed_len_ = (pkt_).get_resp.immed_len; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + immed_len_ = (pkt_).get_accum_resp.immed_len; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + immed_len_ = (pkt_).fop_resp.immed_len; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ @@ -285,7 +299,21 @@ MPIDI_CH3_PKT_DEFS immed_data_ = (pkt_).fop.data; \ break; \ case (MPIDI_CH3_PKT_CAS): \ - /* FIXME: we should deal with CAS here */ \ + /* Note that here we return pointer of origin data, not \ + pointer of compare data. */ \ + immed_data_ = &((pkt_).cas.origin_data); \ + break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + immed_data_ = (pkt_).get_resp.data; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + immed_data_ = (pkt_).get_accum_resp.data; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + immed_data_ = (pkt_).fop_resp.data; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + immed_data_ = &((pkt_).cas_resp.data); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ @@ -374,6 +402,27 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_FOP): \ flags_ = (pkt_).fop.flags; \ break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + flags_ = (pkt_).get_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + flags_ = (pkt_).get_accum_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + flags_ = (pkt_).fop_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + flags_ = (pkt_).cas_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + flags_ = (pkt_).unlock.flags; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + flags_ = (pkt_).lock_ack.flags; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + flags_ = (pkt_).lock_op_ack.flags; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ @@ -401,6 +450,27 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_FOP): \ (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + (pkt_).lock_ack.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + (pkt_).lock_op_ack.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ @@ -428,9 +498,36 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_FOP): \ win_hdl_ = (pkt_).fop.source_win_handle; \ break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + win_hdl_ = (pkt_).get_resp.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + win_hdl_ = (pkt_).get_accum_resp.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + win_hdl_ = (pkt_).fop_resp.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + win_hdl_ = (pkt_).cas_resp.source_win_handle; \ + break; \ case (MPIDI_CH3_PKT_LOCK): \ win_hdl_ = (pkt_).lock.source_win_handle; \ break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + win_hdl_ = (pkt_).unlock.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FLUSH): \ + win_hdl_ = (pkt_).flush.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + win_hdl_ = (pkt_).lock_ack.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + win_hdl_ = (pkt_).lock_op_ack.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FLUSH_ACK): \ + win_hdl_ = (pkt_).flush_ack.source_win_handle; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ @@ -461,6 +558,15 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_LOCK): \ win_hdl_ = (pkt_).lock.target_win_handle; \ break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + win_hdl_ = (pkt_).unlock.target_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FLUSH): \ + win_hdl_ = (pkt_).flush.target_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_DECR_AT_COUNTER): \ + win_hdl_ = (pkt_).decr_at_cnt.target_win_handle; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ http://git.mpich.org/mpich.git/commitdiff/fa95883370a01f71aeabeee548e4076880... commit fa95883370a01f71aeabeee548e4076880ecbd86 Author: Xin Zhao <[email protected]> Date: Mon Feb 9 16:44:25 2015 -0800 Re-apply modifications on mpidpkt.h. This patch re-apply modifications on mpidpkt.h that is temporarily reverted in bb3f962. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 3e0315a..a3e1793 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -86,7 +86,8 @@ typedef enum { MPIDI_CH3_PKT_FOP_RESP, MPIDI_CH3_PKT_CAS_RESP, MPIDI_CH3_PKT_LOCK, - MPIDI_CH3_PKT_LOCK_GRANTED, + MPIDI_CH3_PKT_LOCK_ACK, + MPIDI_CH3_PKT_LOCK_OP_ACK, MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, MPIDI_CH3_PKT_FLUSH_ACK, @@ -502,7 +503,7 @@ typedef struct MPIDI_CH3_Pkt_put { * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; - size_t immed_len; + int immed_len; int lock_type; /* used when piggybacking LOCK message. */ int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_put_t; @@ -533,6 +534,9 @@ typedef struct MPIDI_CH3_Pkt_get_resp { int target_rank; MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; + /* Followings are to piggyback IMMED data */ + int immed_len; + char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_resp_t; typedef struct MPIDI_CH3_Pkt_accum { @@ -551,7 +555,7 @@ typedef struct MPIDI_CH3_Pkt_accum { * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; - size_t immed_len; + int immed_len; int lock_type; /* used when piggybacking LOCK message. */ int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_accum_t; @@ -573,7 +577,7 @@ typedef struct MPIDI_CH3_Pkt_get_accum { * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; - size_t immed_len; + int immed_len; int lock_type; /* used when piggybacking LOCK message. */ int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_accum_t; @@ -585,6 +589,9 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { int target_rank; MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; + /* Followings are to piggyback IMMED data */ + int immed_len; + char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_cas { @@ -648,39 +655,36 @@ typedef struct MPIDI_CH3_Pkt_lock { int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int target_rank; /* Used in unluck/flush response to look up the - * target state at the origin. */ int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { MPIDI_CH3_Pkt_type_t type; - int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int target_rank; /* Used in unluck/flush response to look up the - * target state at the origin. */ - int origin_rank; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_unlock_t; typedef struct MPIDI_CH3_Pkt_flush { MPIDI_CH3_Pkt_type_t type; - int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int target_rank; /* Used in unluck/flush response to look up the - * target state at the origin. */ - int origin_rank; } MPIDI_CH3_Pkt_flush_t; -typedef struct MPIDI_CH3_Pkt_lock_granted { +typedef struct MPIDI_CH3_Pkt_lock_ack { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win source_win_handle; int target_rank; /* Used in flush_ack response to look up the * target state at the origin. */ -} MPIDI_CH3_Pkt_lock_granted_t; +} MPIDI_CH3_Pkt_lock_ack_t; + +typedef struct MPIDI_CH3_Pkt_lock_op_ack { + MPIDI_CH3_Pkt_type_t type; + MPIDI_CH3_Pkt_flags_t flags; + MPI_Win source_win_handle; + int target_rank; +} MPIDI_CH3_Pkt_lock_op_ack_t; typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; @@ -725,7 +729,8 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_accum_t accum; MPIDI_CH3_Pkt_get_accum_t get_accum; MPIDI_CH3_Pkt_lock_t lock; - MPIDI_CH3_Pkt_lock_granted_t lock_granted; + MPIDI_CH3_Pkt_lock_ack_t lock_ack; + MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack; MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; MPIDI_CH3_Pkt_flush_ack_t flush_ack; http://git.mpich.org/mpich.git/commitdiff/2cbc9180045dea815a7aeb7a7d1fe7f299... commit 2cbc9180045dea815a7aeb7a7d1fe7f299020f72 Author: Xin Zhao <[email protected]> Date: Mon Feb 9 16:43:23 2015 -0800 Revert "Code-refactor: arrange RMA pkt structure." This reverts commit 389aab1673b9e54b9172985f69931bf1d731f3d9. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index ed6f4f4..3e0315a 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -86,11 +86,11 @@ typedef enum { MPIDI_CH3_PKT_FOP_RESP, MPIDI_CH3_PKT_CAS_RESP, MPIDI_CH3_PKT_LOCK, + MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, - MPIDI_CH3_PKT_DECR_AT_COUNTER, - MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_FLUSH_ACK, + MPIDI_CH3_PKT_DECR_AT_COUNTER, /* RMA Packets end here */ MPIDI_CH3_PKT_FLOW_CNTL_UPDATE, /* FIXME: Unused */ MPIDI_CH3_PKT_CLOSE, @@ -486,236 +486,214 @@ MPIDI_CH3_PKT_DEFS } \ } -/* RMA packets start here */ - -/********************************************************************************/ -/* RMA packet (from origin to target, including PUT, GET, ACC, GACC, CAS, FOP) */ -/********************************************************************************/ typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ - size_t immed_len; + int dataloop_size; /* for derived datatypes */ + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; + size_t immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_put_t; typedef struct MPIDI_CH3_Pkt_get { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Following is to complete request at origin */ + int dataloop_size; /* for derived datatypes */ MPI_Request request_handle; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_t; +typedef struct MPIDI_CH3_Pkt_get_resp { + MPIDI_CH3_Pkt_type_t type; + MPI_Request request_handle; + /* followings are used to decrement ack_counter at origin */ + int target_rank; + MPI_Win source_win_handle; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_get_resp_t; + typedef struct MPIDI_CH3_Pkt_accum { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Following is to specify ACC op */ + int dataloop_size; /* for derived datatypes */ MPI_Op op; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ - size_t immed_len; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; + size_t immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ + MPI_Request request_handle; /* For get_accumulate response */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Following is to describe ACC op */ + int dataloop_size; /* for derived datatypes */ MPI_Op op; - /* Following is to complete request on origin */ - MPI_Request request_handle; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggback IMMED data */ - size_t immed_len; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; + size_t immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_accum_t; -typedef struct MPIDI_CH3_Pkt_fop { +typedef struct MPIDI_CH3_Pkt_get_accum_resp { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - MPI_Win source_win_handle; - MPI_Win target_win_handle; - /* Followings are to describe target data */ - void *addr; - MPI_Datatype datatype; - /* Following is to speicfy ACC op */ - MPI_Op op; - /* Following is to complete request at origin */ MPI_Request request_handle; - /* Followings are to piggyback IMMED data */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; -} MPIDI_CH3_Pkt_fop_t; + /* followings are used to decrement ack_counter at origin */ + int target_rank; + MPI_Win source_win_handle; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_cas { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win source_win_handle; - MPI_Win target_win_handle; - /* Followings are to describe target data */ - void *addr; MPI_Datatype datatype; - /* Following is to complete request on origin */ + void *addr; MPI_Request request_handle; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ + MPI_Win source_win_handle; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ MPIDI_CH3_CAS_Immed_u origin_data; MPIDI_CH3_CAS_Immed_u compare_data; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_cas_t; - -/*********************************************************************************/ -/* RMA response packet (from target to origin, including GET_RESP, GET_ACC_RESP, */ -/* CAS_RESP, FOP_RESP) */ -/*********************************************************************************/ - -typedef struct MPIDI_CH3_Pkt_get_resp { +typedef struct MPIDI_CH3_Pkt_cas_resp { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ MPI_Request request_handle; - /* TODO: we should add IMMED data here */ - /* Followings are used to decrement ack_counter at origin */ - MPI_Win source_win_handle; + MPIDI_CH3_CAS_Immed_u data; + /* followings are used to decrement ack_counter at orign */ int target_rank; -} MPIDI_CH3_Pkt_get_resp_t; - -typedef struct MPIDI_CH3_Pkt_get_accum_resp { - MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ - MPI_Request request_handle; - /* TODO: we should add IMMED data here */ - /* Followings are used to decrement ack_counter at origin */ MPI_Win source_win_handle; - int target_rank; -} MPIDI_CH3_Pkt_get_accum_resp_t; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_cas_resp_t; -typedef struct MPIDI_CH3_Pkt_fop_resp { +typedef struct MPIDI_CH3_Pkt_fop { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ + MPI_Datatype datatype; + void *addr; + MPI_Op op; MPI_Request request_handle; - /* Followings are used to decrement ack_counter at orign */ MPI_Win source_win_handle; - int target_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; -} MPIDI_CH3_Pkt_fop_resp_t; + int immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ +} MPIDI_CH3_Pkt_fop_t; -typedef struct MPIDI_CH3_Pkt_cas_resp { +typedef struct MPIDI_CH3_Pkt_fop_resp { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ MPI_Request request_handle; - /* Followings are used to decrement ack_counter at orign */ - MPI_Win source_win_handle; + char data[MPIDI_RMA_IMMED_BYTES]; + int immed_len; + /* followings are used to decrement ack_counter at orign */ int target_rank; - /* Following is to piggyback IMMED data */ - MPIDI_CH3_CAS_Immed_u data; -} MPIDI_CH3_Pkt_cas_resp_t; - -/*********************************************************************************/ -/* RMA control packet (from origin to target, including LOCK, UNLOCK, FLUSH) */ -/*********************************************************************************/ + MPI_Win source_win_handle; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_fop_resp_t; typedef struct MPIDI_CH3_Pkt_lock { MPIDI_CH3_Pkt_type_t type; + int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int lock_type; - int target_rank; + int target_rank; /* Used in unluck/flush response to look up the + * target state at the origin. */ int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; + int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int lock_type; - int target_rank; + int target_rank; /* Used in unluck/flush response to look up the + * target state at the origin. */ int origin_rank; + MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_unlock_t; typedef struct MPIDI_CH3_Pkt_flush { MPIDI_CH3_Pkt_type_t type; + int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int lock_type; - int target_rank; + int target_rank; /* Used in unluck/flush response to look up the + * target state at the origin. */ int origin_rank; } MPIDI_CH3_Pkt_flush_t; -typedef struct MPIDI_CH3_Pkt_decr_at_counter { - MPIDI_CH3_Pkt_type_t type; - MPI_Win target_win_handle; -} MPIDI_CH3_Pkt_decr_at_counter_t; - -/*********************************************************************************/ -/* RMA control response packet (from target to origin, including LOCK_GRANTED, */ -/* FLUSH_ACK) */ -/*********************************************************************************/ - typedef struct MPIDI_CH3_Pkt_lock_granted { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win source_win_handle; - int target_rank; + int target_rank; /* Used in flush_ack response to look up the + * target state at the origin. */ } MPIDI_CH3_Pkt_lock_granted_t; typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; MPI_Win source_win_handle; - int target_rank; + int target_rank; /* Used in flush_ack response to look up the + * target state at the origin. */ + MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_flush_ack_t; -/* RMA packets end here */ +typedef struct MPIDI_CH3_Pkt_decr_at_counter { + MPIDI_CH3_Pkt_type_t type; + MPI_Win target_win_handle; +} MPIDI_CH3_Pkt_decr_at_counter_t; typedef struct MPIDI_CH3_Pkt_close { MPIDI_CH3_Pkt_type_t type; @@ -741,25 +719,23 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_rndv_send_t rndv_send; MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req; MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp; - /* RMA packets start here */ MPIDI_CH3_Pkt_put_t put; MPIDI_CH3_Pkt_get_t get; + MPIDI_CH3_Pkt_get_resp_t get_resp; MPIDI_CH3_Pkt_accum_t accum; MPIDI_CH3_Pkt_get_accum_t get_accum; - MPIDI_CH3_Pkt_fop_t fop; - MPIDI_CH3_Pkt_cas_t cas; - MPIDI_CH3_Pkt_get_resp_t get_resp; - MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp; - MPIDI_CH3_Pkt_fop_resp_t fop_resp; - MPIDI_CH3_Pkt_cas_resp_t cas_resp; MPIDI_CH3_Pkt_lock_t lock; + MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; - MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; - MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_flush_ack_t flush_ack; - /* RMA packets end here */ + MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; MPIDI_CH3_Pkt_close_t close; + MPIDI_CH3_Pkt_cas_t cas; + MPIDI_CH3_Pkt_cas_resp_t cas_resp; + MPIDI_CH3_Pkt_fop_t fop; + MPIDI_CH3_Pkt_fop_resp_t fop_resp; + MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp; MPIDI_CH3_Pkt_revoke_t revoke; # if defined(MPIDI_CH3_PKT_DECL) MPIDI_CH3_PKT_DECL http://git.mpich.org/mpich.git/commitdiff/bb3f96236aaab668a2129ec4939976588f... commit bb3f96236aaab668a2129ec4939976588fce4ebf Author: Xin Zhao <[email protected]> Date: Sun Feb 8 19:20:12 2015 -0800 Temporarily revert commits for src/mpid/ch3/include/mpidpkt.h We are going to revert the commit 389aab1 because it re-ordered the attributes in RMA packet structs in mpidpkt.h and messed up the alignments. This commit temporarily reverts the following commits, which only reverts modification on mpidpkt.h after commit 389aab1. e36203c3, 45afd1fd, 3a05784f, 87acbbbe, b155e7e We will re-apply those modifications after we revert 389aab1. Signed-off-by: Pavan Balaji <[email protected]> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 385c99f..ed6f4f4 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -89,8 +89,7 @@ typedef enum { MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, MPIDI_CH3_PKT_DECR_AT_COUNTER, - MPIDI_CH3_PKT_LOCK_ACK, - MPIDI_CH3_PKT_LOCK_OP_ACK, + MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_FLUSH_ACK, /* RMA Packets end here */ MPIDI_CH3_PKT_FLOW_CNTL_UPDATE, /* FIXME: Unused */ @@ -507,7 +506,7 @@ typedef struct MPIDI_CH3_Pkt_put { int lock_type; int origin_rank; /* Followings are to piggyback IMMED data */ - int immed_len; + size_t immed_len; char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_put_t; @@ -544,7 +543,7 @@ typedef struct MPIDI_CH3_Pkt_accum { int lock_type; int origin_rank; /* Followings are to piggyback IMMED data */ - int immed_len; + size_t immed_len; char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_accum_t; @@ -566,7 +565,7 @@ typedef struct MPIDI_CH3_Pkt_get_accum { int lock_type; int origin_rank; /* Followings are to piggback IMMED data */ - int immed_len; + size_t immed_len; char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_accum_t; @@ -623,9 +622,6 @@ typedef struct MPIDI_CH3_Pkt_get_resp { /* Followings are used to decrement ack_counter at origin */ MPI_Win source_win_handle; int target_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_resp_t; typedef struct MPIDI_CH3_Pkt_get_accum_resp { @@ -637,9 +633,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { /* Followings are used to decrement ack_counter at origin */ MPI_Win source_win_handle; int target_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_fop_resp { @@ -676,6 +669,7 @@ typedef struct MPIDI_CH3_Pkt_lock { MPI_Win target_win_handle; MPI_Win source_win_handle; int lock_type; + int target_rank; int origin_rank; } MPIDI_CH3_Pkt_lock_t; @@ -684,12 +678,18 @@ typedef struct MPIDI_CH3_Pkt_unlock { MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; + int lock_type; + int target_rank; + int origin_rank; } MPIDI_CH3_Pkt_unlock_t; typedef struct MPIDI_CH3_Pkt_flush { MPIDI_CH3_Pkt_type_t type; MPI_Win target_win_handle; MPI_Win source_win_handle; + int lock_type; + int target_rank; + int origin_rank; } MPIDI_CH3_Pkt_flush_t; typedef struct MPIDI_CH3_Pkt_decr_at_counter { @@ -698,23 +698,16 @@ typedef struct MPIDI_CH3_Pkt_decr_at_counter { } MPIDI_CH3_Pkt_decr_at_counter_t; /*********************************************************************************/ -/* RMA control response packet (from target to origin, including LOCK_ACK, */ -/* LOCK_OP_ACK, FLUSH_ACK) */ +/* RMA control response packet (from target to origin, including LOCK_GRANTED, */ +/* FLUSH_ACK) */ /*********************************************************************************/ -typedef struct MPIDI_CH3_Pkt_lock_ack { - MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - MPI_Win source_win_handle; - int target_rank; -} MPIDI_CH3_Pkt_lock_ack_t; - -typedef struct MPIDI_CH3_Pkt_lock_op_ack { +typedef struct MPIDI_CH3_Pkt_lock_granted { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win source_win_handle; int target_rank; -} MPIDI_CH3_Pkt_lock_op_ack_t; +} MPIDI_CH3_Pkt_lock_granted_t; typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; @@ -763,8 +756,7 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; - MPIDI_CH3_Pkt_lock_ack_t lock_ack; - MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack; + MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_flush_ack_t flush_ack; /* RMA packets end here */ MPIDI_CH3_Pkt_close_t close; ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/include/mpid_rma_issue.h | 151 ++------ src/mpid/ch3/include/mpid_rma_lockqueue.h | 3 +- src/mpid/ch3/include/mpid_rma_types.h | 1 + src/mpid/ch3/include/mpidimpl.h | 2 + src/mpid/ch3/include/mpidpkt.h | 613 ++++++++++++++++------------- src/mpid/ch3/include/mpidpre.h | 1 - src/mpid/ch3/include/mpidrma.h | 141 ++++--- src/mpid/ch3/src/ch3u_handle_recv_pkt.c | 18 +- src/mpid/ch3/src/ch3u_handle_recv_req.c | 509 +++++++++++++++++-------- src/mpid/ch3/src/ch3u_handle_send_req.c | 20 +- src/mpid/ch3/src/ch3u_request.c | 1 - src/mpid/ch3/src/ch3u_rma_oplist.c | 18 +- src/mpid/ch3/src/ch3u_rma_ops.c | 460 +++++++++++++++------- src/mpid/ch3/src/ch3u_rma_pkthandler.c | 555 ++++++++++++++++++-------- test/mpi/rma/atomic_rmw_gacc.c | 12 +- 15 files changed, 1574 insertions(+), 931 deletions(-) hooks/post-receive -- MPICH primary repository