February 2015 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, mpich-master, created. v3.2a2-163-gd9c15cf
by noreply＠mpich.org 13 Feb '15

13 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, mpich-master has been created at d9c15cf3a93674c129d94ecb5b2e2b13bc186ff9 (commit) - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/d9c15cf3a93674c129d94ecb5b2e2b13b… commit d9c15cf3a93674c129d94ecb5b2e2b13bc186ff9 Author: Sameh Sharkawi <sssharka(a)us.ibm.com> Date: Wed Jan 21 13:30:50 2015 -0500 PAMID: Initial CUDA support This is an initial limited implementation for CUDA support. This is not performance optimized and only for testing. (ibm) D202477 Signed-off-by: Su Huang <suhuang(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 7c6e566..1324187 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -38,6 +38,11 @@ #include "opa_primitives.h" +#if CUDA_AWARE_SUPPORT +#include <cuda_runtime_api.h> +#endif + + #if (MPIU_HANDLE_ALLOCATION_METHOD == MPIU_HANDLE_ALLOCATION_THREAD_LOCAL) && defined(__BGQ__) struct MPID_Request; typedef struct @@ -105,6 +110,11 @@ typedef struct #if QUEUE_BINARY_SEARCH_SUPPORT unsigned queue_binary_search_support_on; #endif + +#if CUDA_AWARE_SUPPORT + unsigned cuda_aware_support_on; +#endif + unsigned verbose; /**< The current level of verbosity for end-of-job stats. */ unsigned statistics; /**< The current level of stats collection. */ unsigned rma_pending; /**< The max num outstanding requests during an RMA op */ diff --git a/src/mpid/pamid/include/mpidi_platform.h b/src/mpid/pamid/include/mpidi_platform.h index e5ac1d8..cd95e72 100644 --- a/src/mpid/pamid/include/mpidi_platform.h +++ b/src/mpid/pamid/include/mpidi_platform.h @@ -86,6 +86,7 @@ typedef int32_t MPID_Node_id_t; #define PAMIX_IS_LOCAL_TASK_SHIFT (6) #define MPIDI_SMP_DETECT_DEFAULT 1 #define TOKEN_FLOW_CONTROL 0 +#define CUDA_AWARE_SUPPORT 0 /* * Enable both the 'internal vs application' and the 'local vs remote' @@ -142,6 +143,7 @@ static const char _ibm_release_version_[] = "V1R2M0"; #define MPIDI_NO_ASSERT 1 #define TOKEN_FLOW_CONTROL 1 #define DYNAMIC_TASKING 1 +#define CUDA_AWARE_SUPPORT 1 /* 'is local task' extension and limits */ #define PAMIX_IS_LOCAL_TASK diff --git a/src/mpid/pamid/include/mpidi_prototypes.h b/src/mpid/pamid/include/mpidi_prototypes.h index d833637..083699e 100644 --- a/src/mpid/pamid/include/mpidi_prototypes.h +++ b/src/mpid/pamid/include/mpidi_prototypes.h @@ -266,6 +266,7 @@ pami_result_t MPIDI_Register_algorithms_ext(void *cookie, size_t *num_algorithms); int MPIDI_collsel_pami_tune_parse_params(int argc, char ** argv); void MPIDI_collsel_pami_tune_cleanup(); +inline bool MPIDI_cuda_is_device_buf(const void* ptr); void MPIDI_Coll_Comm_create (MPID_Comm *comm); void MPIDI_Coll_Comm_destroy(MPID_Comm *comm); void MPIDI_Comm_coll_query (MPID_Comm *comm); diff --git a/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c b/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c index 5920817..75d2a9c 100644 --- a/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c +++ b/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c @@ -117,6 +117,22 @@ int MPIDO_Allreduce(const void *sendbuf, fprintf(stderr,"Using MPICH allreduce type %u.\n", selected_type); MPIDI_Update_last_algorithm(comm_ptr, "ALLREDUCE_MPICH"); +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(sendbuf)) + { + MPI_Aint dt_extent; + MPID_Datatype_get_extent_macro(dt, dt_extent); + char *buf = MPIU_Malloc(dt_extent * count); + cudaError_t cudaerr = cudaMemcpy(buf, sendbuf, dt_extent * count, cudaMemcpyDeviceToHost); + if (cudaSuccess != cudaerr) { + fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(cudaerr)); + } + int cuda_res = MPIR_Allreduce(buf, recvbuf, count, dt, op, comm_ptr, mpierrno); + MPIU_Free(buf); + return cuda_res; + } + else +#endif return MPIR_Allreduce(sendbuf, recvbuf, count, dt, op, comm_ptr, mpierrno); } diff --git a/src/mpid/pamid/src/coll/reduce/mpido_reduce.c b/src/mpid/pamid/src/coll/reduce/mpido_reduce.c index d3ac54b..2aeab7b 100644 --- a/src/mpid/pamid/src/coll/reduce/mpido_reduce.c +++ b/src/mpid/pamid/src/coll/reduce/mpido_reduce.c @@ -117,6 +117,22 @@ int MPIDO_Reduce(const void *sendbuf, { if(unlikely(verbose)) fprintf(stderr,"Using MPICH reduce algorithm\n"); +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(sendbuf)) + { + MPI_Aint dt_extent; + MPID_Datatype_get_extent_macro(datatype, dt_extent); + char *buf = MPIU_Malloc(dt_extent * count); + cudaError_t cudaerr = cudaMemcpy(buf, sendbuf, dt_extent * count, cudaMemcpyDeviceToHost); + if (cudaSuccess != cudaerr) { + fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(cudaerr)); + } + int cuda_res = MPIR_Reduce(buf, recvbuf, count, datatype, op, root, comm_ptr, mpierrno); + MPIU_Free(buf); + return cuda_res; + } + else +#endif return MPIR_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, mpierrno); } diff --git a/src/mpid/pamid/src/coll/scan/mpido_scan.c b/src/mpid/pamid/src/coll/scan/mpido_scan.c index 6f33c02..c0100c7 100644 --- a/src/mpid/pamid/src/coll/scan/mpido_scan.c +++ b/src/mpid/pamid/src/coll/scan/mpido_scan.c @@ -135,6 +135,26 @@ int MPIDO_Doscan(const void *sendbuf, void *recvbuf, { if(unlikely(verbose)) fprintf(stderr,"Using MPICH scan algorithm (exflag %d)\n",exflag); +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(sendbuf)) + { + MPI_Aint dt_extent; + MPID_Datatype_get_extent_macro(datatype, dt_extent); + char *buf = MPIU_Malloc(dt_extent * count); + cudaError_t cudaerr = cudaMemcpy(buf, sendbuf, dt_extent * count, cudaMemcpyDeviceToHost); + if (cudaSuccess != cudaerr) { + fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(cudaerr)); + } + int cuda_res; + if(exflag) + cuda_res = MPIR_Exscan(buf, recvbuf, count, datatype, op, comm_ptr, mpierrno); + else + cuda_res = MPIR_Scan(buf, recvbuf, count, datatype, op, comm_ptr, mpierrno); + MPIU_Free(buf); + return cuda_res; + } + else +#endif if(exflag) return MPIR_Exscan(sendbuf, recvbuf, count, datatype, op, comm_ptr, mpierrno); else diff --git a/src/mpid/pamid/src/mpid_buffer.c b/src/mpid/pamid/src/mpid_buffer.c index 884b92c..0e2ad5a 100644 --- a/src/mpid/pamid/src/mpid_buffer.c +++ b/src/mpid/pamid/src/mpid_buffer.c @@ -57,10 +57,12 @@ void MPIDI_Buffer_copy( MPID_Datatype * sdt_ptr; MPID_Datatype * rdt_ptr; + MPI_Aint sdt_extent; + MPI_Aint rdt_extent; + *smpi_errno = MPI_SUCCESS; *rmpi_errno = MPI_SUCCESS; - /* printf("bufcopy: src count=%d dt=%d\n", scount, sdt); */ /* printf("bufcopy: dst count=%d dt=%d\n", rcount, rdt); */ @@ -83,11 +85,52 @@ void MPIDI_Buffer_copy( if (sdt_contig && rdt_contig) { +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rbuf)) + { + cudaError_t cudaerr = cudaMemcpy(rbuf + rdt_true_lb, sbuf + sdt_true_lb, sdata_sz, cudaMemcpyHostToDevice); + } + else +#endif memcpy((char*)rbuf + rdt_true_lb, (const char *)sbuf + sdt_true_lb, sdata_sz); *rsz = sdata_sz; } else if (sdt_contig) { +#if CUDA_AWARE_SUPPORT + // This will need to be done in two steps: + // 1 - Allocate a temp buffer which is the same size as user buffer and unpack in it. + // 2 - Copy unpacked data into user buffer from temp buffer. + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rbuf)) + { + MPID_Datatype_get_extent_macro(rdt, rdt_extent); + char *buf = MPIU_Malloc(rdt_extent * rcount); + memset(buf, 0, rdt_extent * rcount); + MPID_Segment seg; + DLOOP_Offset last; + + MPID_Segment_init(buf, rcount, rdt, &seg, 0); + last = sdata_sz; + MPID_Segment_unpack(&seg, 0, &last, (char*)sbuf + sdt_true_lb); + /* --BEGIN ERROR HANDLING-- */ + if (last != sdata_sz) + { + *rmpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, __FUNCTION__, __LINE__, MPI_ERR_TYPE, "**dtypemismatch", 0); + } + /* --END ERROR HANDLING-- */ + + *rsz = last; + + + cudaError_t cudaerr = cudaMemcpy(rbuf + rdt_true_lb, buf, rdt_extent * rcount, cudaMemcpyHostToDevice); + + MPIU_Free(buf); + + goto fn_exit; + + } +#endif + MPID_Segment seg; DLOOP_Offset last; diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 52f0b7c..e768119 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -119,6 +119,9 @@ MPIDI_Process_t MPIDI_Process = { #ifdef QUEUE_BINARY_SEARCH_SUPPORT .queue_binary_search_support_on = 0, #endif +#if CUDA_AWARE_SUPPORT + .cuda_aware_support_on = 0, +#endif .rma_pending = 1000, .shmem_pt2pt = 1, .smp_detect = MPIDI_SMP_DETECT_DEFAULT, diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c index dd1912f..c931858 100644 --- a/src/mpid/pamid/src/mpidi_env.c +++ b/src/mpid/pamid/src/mpidi_env.c @@ -1139,6 +1139,12 @@ MPIDI_Env_setup(int rank, int requested) char* names[] = {"MP_S_USE_QUEUE_BINARY_SEARCH_SUPPORT", NULL}; ENV_Char(names, &MPIDI_Process.queue_binary_search_support_on); #endif + +#if CUDA_AWARE_SUPPORT + char* names[] = {"MP_CUDA_AWARE", NULL}; + ENV_Char(names, &MPIDI_Process.cuda_aware_support_on); +#endif + /* Exit if any deprecated environment variables were specified. */ if (found_deprecated_env_var) { diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c index 774e786..896dd86 100644 --- a/src/mpid/pamid/src/mpidi_util.c +++ b/src/mpid/pamid/src/mpidi_util.c @@ -1921,6 +1921,34 @@ void MPIDI_collsel_pami_tune_cleanup() MPIDI_collsel_free_advisor_params(&MPIDI_Collsel_advisor_params); } + + +/**********************************************************/ +/* CUDA Utilities */ +/**********************************************************/ + +inline bool MPIDI_cuda_is_device_buf(const void* ptr) +{ + bool result = false; +#if CUDA_AWARE_SUPPORT + struct cudaPointerAttributes cuda_attr; + cudaError_t e= cudaPointerGetAttributes ( & cuda_attr, ptr); + + if (e != cudaSuccess) + result = false; + else if (cuda_attr.memoryType == cudaMemoryTypeDevice) + result = true; + else + result = false; +#endif + return result; +} + + +/**********************************************************/ +/* End CUDA Utilities */ +/**********************************************************/ + #if defined(MPID_USE_NODE_IDS) #undef FUNCNAME #define FUNCNAME MPID_Get_node_id diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c index 705d623..1746c45 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c @@ -195,7 +195,16 @@ MPIDI_RecvShortCB(pami_context_t context, void* rcvbuf = rreq->mpid.userbuf; if (sndlen > 0) - memcpy(rcvbuf, sndbuf, sndlen); + { +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf)) + { + cudaError_t cudaerr = cudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice); + } + else +#endif + memcpy(rcvbuf, sndbuf, sndlen); + } TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen); TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH); TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c index fa3b48b..c91e7da 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c @@ -175,7 +175,13 @@ MPIDI_Callback_process_userdefined_dt(pami_context_t context, MPID_assert(rreq->mpid.uebuf == NULL); MPID_assert(rreq->mpid.uebuflen == 0); void* rcvbuf = rreq->mpid.userbuf + dt_true_lb;; - +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf)) + { + cudaError_t cudaerr = cudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice); + } + else +#endif memcpy(rcvbuf, sndbuf, sndlen); MPIDI_Request_complete(rreq); return; diff --git a/src/mpid/pamid/src/pt2pt/mpidi_rendezvous.c b/src/mpid/pamid/src/pt2pt/mpidi_rendezvous.c index ec4569b..d7f0a0c 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_rendezvous.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_rendezvous.c @@ -107,6 +107,16 @@ MPIDI_RendezvousTransfer(pami_context_t context, pami_endpoint_t dest; MPIDI_Context_endpoint(rreq, &dest); +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf)) + { + MPIDI_RendezvousTransfer_use_pami_get(context,dest,rcvbuf,rreq); + } + else + { +#endif + + #ifdef USE_PAMI_RDMA size_t rcvlen_out; rc = PAMI_Memregion_create(context, @@ -153,6 +163,10 @@ MPIDI_RendezvousTransfer(pami_context_t context, } #endif +#if CUDA_AWARE_SUPPORT + } +#endif + return PAMI_SUCCESS; } diff --git a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c index 2086e43..65cf773 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c @@ -345,6 +345,26 @@ MPIDI_SendMsg_process_userdefined_dt(MPID_Request * sreq, */ else { + char *buf = NULL; +#if CUDA_AWARE_SUPPORT + // This will need to be done in two steps: + // 1 - Allocate a temp buffer which is the same size as user buffer and copy in it. + // 2 - Pack data into ue buffer from temp buffer. + int on_device = MPIDI_cuda_is_device_buf(sreq->mpid.userbuf); + if(MPIDI_Process.cuda_aware_support_on && on_device) + { + MPI_Aint dt_extent; + MPID_Datatype_get_extent_macro(sreq->mpid.datatype, dt_extent); + buf = MPIU_Malloc(dt_extent * sreq->mpid.userbufcount); + + cudaError_t cudaerr = cudaMemcpy(buf, sreq->mpid.userbuf, dt_extent * sreq->mpid.userbufcount, cudaMemcpyDeviceToHost); + if (cudaSuccess != cudaerr) { + fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(cudaerr)); + } + + } +#endif + MPID_Segment segment; if(data_sz != 0) { @@ -359,13 +379,23 @@ MPIDI_SendMsg_process_userdefined_dt(MPID_Request * sreq, sreq->mpid.uebuf_malloc = mpiuMalloc; DLOOP_Offset last = data_sz; - MPID_Segment_init(sreq->mpid.userbuf, +#if CUDA_AWARE_SUPPORT + if(!MPIDI_Process.cuda_aware_support_on || !on_device) + buf = sreq->mpid.userbuf; +#endif + MPID_assert(buf != NULL); + + MPID_Segment_init(buf, sreq->mpid.userbufcount, sreq->mpid.datatype, &segment, 0); MPID_Segment_pack(&segment, 0, &last, sndbuf); MPID_assert(last == data_sz); +#if CUDA_AWARE_SUPPORT + if(MPIDI_Process.cuda_aware_support_on && on_device) + MPIU_Free(buf); +#endif } else { sndbuf = NULL; } ----------------------------------------------------------------------- hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-162-g3df5868
by noreply＠mpich.org 13 Feb '15

13 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3df5868917cae81c72f098100f80ff2f79985905 (commit) via 21126e9e6a565575078b89e7cb563bb2aec2c5a9 (commit) via e3ccad1f165fc730aad3a6bd5b6338a9c42b1087 (commit) via e3fc7e702299543e7d6e9d07a169ef71517a7208 (commit) via 80a71e113f179cd13eef8f0e9c7481332df06d72 (commit) via 42c52518e0179196e8dd8021f6e72cbaa9a417fd (commit) via c8ecef8d531601900e9d8db58284897711295d12 (commit) via 59afc29cbf253c7cb7f8efb052e816338cb99304 (commit) via 1b30ab19a3dfdd6ae31c87dba1928ee68ec2775a (commit) via 21479b00b1080d577be8ff7ed66a7bdd62dd16ac (commit) via de9d0f21b33691ee555d3479acdc5e5863af92ca (commit) via 3a017faae893f03d2ad2fb4d24f037d964782e23 (commit) via 81e2b2742b9a690e68d40d93e42a6d3684206fa3 (commit) via d46b848a7e574a92521abe02bb63ba9275512d14 (commit) via 064e60ce4e6903c56c27db5d6d8790f4be05f491 (commit) via fa95883370a01f71aeabeee548e4076880ecbd86 (commit) via 2cbc9180045dea815a7aeb7a7d1fe7f299020f72 (commit) via bb3f96236aaab668a2129ec4939976588fce4ebf (commit) from b21dc461115bdf99d42176ecea8e8efc763d8a2d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3df5868917cae81c72f098100f80ff2f7… commit 3df5868917cae81c72f098100f80ff2f79985905 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Wed Feb 11 02:15:17 2015 -0600 Bug-fix: add barriers between init buffers and issuing RMA operations. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/test/mpi/rma/atomic_rmw_gacc.c b/test/mpi/rma/atomic_rmw_gacc.c index d04e9c3..8bd7582 100644 --- a/test/mpi/rma/atomic_rmw_gacc.c +++ b/test/mpi/rma/atomic_rmw_gacc.c @@ -149,7 +149,9 @@ int main (int argc, char *argv[]) { MPI_Win_unlock(rank, win); } - MPI_Win_lock_all(0, win); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); if (rank != dest) { for (i = 0; i < my_buf_num; i++) { MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), 1, origin_dtp, @@ -176,7 +178,9 @@ int main (int argc, char *argv[]) { MPI_Win_unlock(rank, win); } - MPI_Win_lock_all(0, win); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); if (rank != dest) { for (i = 0; i < my_buf_num; i++) { MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), OP_COUNT, MPI_INT, @@ -203,7 +207,9 @@ int main (int argc, char *argv[]) { MPI_Win_unlock(rank, win); } - MPI_Win_lock_all(0, win); + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); if (rank != dest) { for (i = 0; i < my_buf_num; i++) { MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), 1, origin_dtp, http://git.mpich.org/mpich.git/commitdiff/21126e9e6a565575078b89e7cb563bb2a… commit 21126e9e6a565575078b89e7cb563bb2aec2c5a9 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 13:51:03 2015 -0800 Delete comments that no longer make sense. The comments are no longer significant for new RMA infrastructure. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 699a1f3..aa2db5b 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -591,13 +591,8 @@ typedef struct MPIDI_CH3_Pkt_put { void *addr; int count; MPI_Datatype datatype; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ + MPI_Win target_win_handle; + MPI_Win source_win_handle; union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -617,10 +612,7 @@ typedef struct MPIDI_CH3_Pkt_get { int dataloop_size; /* for derived datatypes */ } info; MPI_Request request_handle; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; } MPIDI_CH3_Pkt_get_t; typedef struct MPIDI_CH3_Pkt_get_resp { @@ -645,13 +637,8 @@ typedef struct MPIDI_CH3_Pkt_accum { int count; MPI_Datatype datatype; MPI_Op op; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ + MPI_Win target_win_handle; + MPI_Win source_win_handle; union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -666,10 +653,7 @@ typedef struct MPIDI_CH3_Pkt_get_accum { int count; MPI_Datatype datatype; MPI_Op op; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -697,10 +681,7 @@ typedef struct MPIDI_CH3_Pkt_cas { MPI_Datatype datatype; void *addr; MPI_Request request_handle; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; MPIDI_CH3_CAS_Immed_u origin_data; MPIDI_CH3_CAS_Immed_u compare_data; } MPIDI_CH3_Pkt_cas_t; @@ -726,10 +707,7 @@ typedef struct MPIDI_CH3_Pkt_fop { void *addr; MPI_Op op; MPI_Request request_handle; - MPI_Win target_win_handle; /* Used in the last RMA operation in each - * epoch for decrementing rma op counter in - * active target rma and for unlocking window - * in passive target rma. Otherwise set to NULL*/ + MPI_Win target_win_handle; struct { /* note that we use struct here in order to consistently access data @@ -786,8 +764,7 @@ typedef struct MPIDI_CH3_Pkt_lock_ack { or request_handle is used. */ MPI_Win source_win_handle; MPI_Request request_handle; - int target_rank; /* Used in flush_ack response to look up the - * target state at the origin. */ + int target_rank; } MPIDI_CH3_Pkt_lock_ack_t; typedef struct MPIDI_CH3_Pkt_lock_op_ack { @@ -803,8 +780,7 @@ typedef struct MPIDI_CH3_Pkt_lock_op_ack { typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; MPI_Win source_win_handle; - int target_rank; /* Used in flush_ack response to look up the - * target state at the origin. */ + int target_rank; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_flush_ack_t; http://git.mpich.org/mpich.git/commitdiff/e3ccad1f165fc730aad3a6bd5b6338a9c… commit e3ccad1f165fc730aad3a6bd5b6338a9c42b1087 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 12:00:25 2015 -0800 Delete unnecessary code. Here req->dev.user_count is used when receiving FOP/CAS response data on origin in PktHandler_FOPResp and PktHandler_CASResp. Since the count always be 1, we did not set rma_op->result_count, and we directly set req->dev.user_count to 1 in packet handlers. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index eda0085..47b2569 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -667,7 +667,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, MPIU_Object_set_ref(rma_op->request, 2); rma_op->request->dev.user_buf = rma_op->result_addr; - rma_op->request->dev.user_count = rma_op->result_count; rma_op->request->dev.datatype = rma_op->result_datatype; rma_op->request->dev.target_win_handle = MPI_WIN_NULL; @@ -733,7 +732,6 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, MPIU_Object_set_ref(resp_req, 2); resp_req->dev.user_buf = rma_op->result_addr; - resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; resp_req->dev.target_win_handle = MPI_WIN_NULL; resp_req->dev.source_win_handle = win_ptr->handle; http://git.mpich.org/mpich.git/commitdiff/e3fc7e702299543e7d6e9d07a169ef715… commit e3fc7e702299543e7d6e9d07a169ef71517a7208 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 11:52:19 2015 -0800 Simplify code of issuing RMA packets. When issuing RMA packets, we do not need to store target_win_handle in the request on origin side but only need to store source_win_handle. Because when the response data is back, we only needs to use source_win_handle on origin size. This patch simplifies the code in this way. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 7cb4052..eda0085 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -457,7 +457,7 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, resp_req->dev.user_buf = rma_op->result_addr; resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; + resp_req->dev.target_win_handle = MPI_WIN_NULL; resp_req->dev.source_win_handle = win_ptr->handle; if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { @@ -670,7 +670,7 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, rma_op->request->dev.user_count = rma_op->result_count; rma_op->request->dev.datatype = rma_op->result_datatype; - rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; + rma_op->request->dev.target_win_handle = MPI_WIN_NULL; rma_op->request->dev.source_win_handle = win_ptr->handle; cas_pkt->request_handle = rma_op->request->handle; @@ -735,7 +735,7 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, resp_req->dev.user_buf = rma_op->result_addr; resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; - resp_req->dev.target_win_handle = fop_pkt->target_win_handle; + resp_req->dev.target_win_handle = MPI_WIN_NULL; resp_req->dev.source_win_handle = win_ptr->handle; fop_pkt->request_handle = resp_req->handle; http://git.mpich.org/mpich.git/commitdiff/80a71e113f179cd13eef8f0e9c7481332… commit 80a71e113f179cd13eef8f0e9c7481332df06d72 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 11:37:50 2015 -0800 Remove source_win_handle from GET-like RMA packets. For GET-like RMA packets and response packets (GACC, GET, FOP, CAS, GACC_RESP, GET_RESP, FOP_RESP, CAS_RESP), originally we carry source_win_handle in packet struct in order to locate window handle on origin side in the packet handler of response packets. However, this is not necessary because source_win_handle can be stored in the request on the origin side. This patch delete source_win_handle from those packets to reduce the size of packet union. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index 2c1a821..7cb4052 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -458,7 +458,7 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; - resp_req->dev.source_win_handle = get_accum_pkt->source_win_handle; + resp_req->dev.source_win_handle = win_ptr->handle; if (!MPIR_DATATYPE_IS_PREDEFINED(resp_req->dev.datatype)) { MPID_Datatype *result_dtp = NULL; @@ -572,7 +572,7 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, rma_op->request->dev.user_count = rma_op->origin_count; rma_op->request->dev.datatype = rma_op->origin_datatype; rma_op->request->dev.target_win_handle = MPI_WIN_NULL; - rma_op->request->dev.source_win_handle = get_pkt->source_win_handle; + rma_op->request->dev.source_win_handle = win_ptr->handle; if (!MPIR_DATATYPE_IS_PREDEFINED(rma_op->request->dev.datatype)) { MPID_Datatype_get_ptr(rma_op->request->dev.datatype, dtp); rma_op->request->dev.datatype_ptr = dtp; @@ -671,7 +671,7 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, rma_op->request->dev.datatype = rma_op->result_datatype; rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; - rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle; + rma_op->request->dev.source_win_handle = win_ptr->handle; cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags |= flags; @@ -736,7 +736,7 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, resp_req->dev.user_count = rma_op->result_count; resp_req->dev.datatype = rma_op->result_datatype; resp_req->dev.target_win_handle = fop_pkt->target_win_handle; - resp_req->dev.source_win_handle = fop_pkt->source_win_handle; + resp_req->dev.source_win_handle = win_ptr->handle; fop_pkt->request_handle = resp_req->handle; diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index cbd300c..699a1f3 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -442,39 +442,10 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_PUT_IMMED): \ win_hdl_ = (pkt_).put.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_GET): \ - win_hdl_ = (pkt_).get.source_win_handle; \ - break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ win_hdl_ = (pkt_).accum.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ - win_hdl_ = (pkt_).get_accum.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_CAS_IMMED): \ - win_hdl_ = (pkt_).cas.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - case (MPIDI_CH3_PKT_FOP_IMMED): \ - win_hdl_ = (pkt_).fop.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_GET_RESP): \ - case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ - win_hdl_ = (pkt_).get_resp.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ - win_hdl_ = (pkt_).get_accum_resp.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_FOP_RESP): \ - case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ - win_hdl_ = (pkt_).fop_resp.source_win_handle; \ - break; \ - case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ - win_hdl_ = (pkt_).cas_resp.source_win_handle; \ - break; \ case (MPIDI_CH3_PKT_LOCK): \ win_hdl_ = (pkt_).lock.source_win_handle; \ break; \ @@ -567,6 +538,53 @@ MPIDI_CH3_PKT_DEFS } \ } +#define MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE(pkt_, request_hdl_, err_) \ + { \ + err_ = MPI_SUCCESS; \ + switch((pkt_).type) { \ + case (MPIDI_CH3_PKT_GET): \ + request_hdl_ = (pkt_).get.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ + request_hdl_ = (pkt_).get_accum.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ + request_hdl_ = (pkt_).cas.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ + request_hdl_ = (pkt_).fop.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ + request_hdl_ = (pkt_).get_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ + request_hdl_ = (pkt_).get_accum_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ + request_hdl_ = (pkt_).fop_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ + request_hdl_ = (pkt_).cas_resp.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK): \ + request_hdl_ = (pkt_).lock.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + request_hdl_ = (pkt_).lock_ack.request_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + request_hdl_ = (pkt_).lock_op_ack.request_handle; \ + break; \ + default: \ + MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ + } \ + } + typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; @@ -603,9 +621,6 @@ typedef struct MPIDI_CH3_Pkt_get { * epoch for decrementing rma op counter in * active target rma and for unlocking window * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ } MPIDI_CH3_Pkt_get_t; typedef struct MPIDI_CH3_Pkt_get_resp { @@ -613,7 +628,6 @@ typedef struct MPIDI_CH3_Pkt_get_resp { MPI_Request request_handle; /* followings are used to decrement ack_counter at origin */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ struct { @@ -656,9 +670,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum { * epoch for decrementing rma op counter in * active target rma and for unlocking window * in passive target rma. Otherwise set to NULL*/ - MPI_Win source_win_handle; /* Used in the last RMA operation in an - * epoch in the case of passive target rma - * with shared locks. Otherwise set to NULL*/ union { int dataloop_size; char data[MPIDI_RMA_IMMED_BYTES]; @@ -670,7 +681,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { MPI_Request request_handle; /* followings are used to decrement ack_counter at origin */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ struct { @@ -687,7 +697,6 @@ typedef struct MPIDI_CH3_Pkt_cas { MPI_Datatype datatype; void *addr; MPI_Request request_handle; - MPI_Win source_win_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -707,7 +716,6 @@ typedef struct MPIDI_CH3_Pkt_cas_resp { } info; /* followings are used to decrement ack_counter at orign */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_cas_resp_t; @@ -718,7 +726,6 @@ typedef struct MPIDI_CH3_Pkt_fop { void *addr; MPI_Op op; MPI_Request request_handle; - MPI_Win source_win_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -742,7 +749,6 @@ typedef struct MPIDI_CH3_Pkt_fop_resp { } info; /* followings are used to decrement ack_counter at orign */ int target_rank; - MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_fop_resp_t; @@ -750,7 +756,14 @@ typedef struct MPIDI_CH3_Pkt_lock { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; + /* Note that either source_win_handle + or request_handle will be used. Here + we need both of them because PUT/GET + may be converted to LOCK packet, + PUT has source_win_handle area and + GET has request_handle area. */ MPI_Win source_win_handle; + MPI_Request request_handle; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { @@ -769,7 +782,10 @@ typedef struct MPIDI_CH3_Pkt_flush { typedef struct MPIDI_CH3_Pkt_lock_ack { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; + /* note that either source_win_handle + or request_handle is used. */ MPI_Win source_win_handle; + MPI_Request request_handle; int target_rank; /* Used in flush_ack response to look up the * target state at the origin. */ } MPIDI_CH3_Pkt_lock_ack_t; @@ -777,7 +793,10 @@ typedef struct MPIDI_CH3_Pkt_lock_ack { typedef struct MPIDI_CH3_Pkt_lock_op_ack { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; + /* note that either source_win_handle + or request_handle is used. */ MPI_Win source_win_handle; + MPI_Request request_handle; int target_rank; } MPIDI_CH3_Pkt_lock_op_ack_t; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 1b4fdf9..c60124a 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -32,6 +32,7 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; lock_pkt->source_win_handle = win_ptr->handle; + lock_pkt->request_handle = MPI_REQUEST_NULL; lock_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (lock_type == MPI_LOCK_SHARED) lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; @@ -110,7 +111,8 @@ static inline int send_unlock_msg(int dest, MPID_Win * win_ptr, #define FCNAME MPIDI_QUOTE(FUNCNAME) static inline int MPIDI_CH3I_Send_lock_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle) + MPI_Win source_win_handle, + MPI_Request request_handle) { MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_ack_t *lock_ack_pkt = &upkt.lock_ack; @@ -120,9 +122,12 @@ static inline int MPIDI_CH3I_Send_lock_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_p MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_ACK_PKT); + MPIU_Assert(!(source_win_handle != MPI_WIN_NULL && request_handle != MPI_REQUEST_NULL)); + /* send lock ack packet */ MPIDI_Pkt_init(lock_ack_pkt, MPIDI_CH3_PKT_LOCK_ACK); lock_ack_pkt->source_win_handle = source_win_handle; + lock_ack_pkt->request_handle = request_handle; lock_ack_pkt->target_rank = win_ptr->comm_ptr->rank; lock_ack_pkt->flags = flags; @@ -153,7 +158,8 @@ static inline int MPIDI_CH3I_Send_lock_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_p #define FCNAME MPIDI_QUOTE(FUNCNAME) static inline int MPIDI_CH3I_Send_lock_op_ack_pkt(MPIDI_VC_t * vc, MPID_Win * win_ptr, MPIDI_CH3_Pkt_flags_t flags, - MPI_Win source_win_handle) + MPI_Win source_win_handle, + MPI_Request request_handle) { MPIDI_CH3_Pkt_t upkt; MPIDI_CH3_Pkt_lock_op_ack_t *lock_op_ack_pkt = &upkt.lock_op_ack; @@ -163,9 +169,12 @@ static inline int MPIDI_CH3I_Send_lock_op_ack_pkt(MPIDI_VC_t * vc, MPID_Win * wi MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_LOCK_OP_ACK_PKT); + MPIU_Assert(!(source_win_handle != MPI_WIN_NULL && request_handle != MPI_REQUEST_NULL)); + /* send lock ack packet */ MPIDI_Pkt_init(lock_op_ack_pkt, MPIDI_CH3_PKT_LOCK_OP_ACK); lock_op_ack_pkt->source_win_handle = source_win_handle; + lock_op_ack_pkt->request_handle = request_handle; lock_op_ack_pkt->target_rank = win_ptr->comm_ptr->rank; lock_op_ack_pkt->flags = flags; @@ -318,6 +327,7 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_RMA_Lock_entry_t *new_ptr = NULL; MPIDI_CH3_Pkt_flags_t flag; MPI_Win source_win_handle; + MPI_Request request_handle; int lock_discarded = 0, data_discarded = 0; int mpi_errno = MPI_SUCCESS; @@ -385,12 +395,21 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Pkt_flags_t flags; MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE((*pkt), target_win_handle, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); MPIDI_CH3_PKT_RMA_GET_FLAGS((*pkt), flags, mpi_errno); + if (pkt->type == MPIDI_CH3_PKT_PUT || pkt->type == MPIDI_CH3_PKT_ACCUMULATE) { + MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); + request_handle = MPI_REQUEST_NULL; + } + else { + source_win_handle = MPI_WIN_NULL; + MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE((*pkt), request_handle, mpi_errno); + } + MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = target_win_handle; lock_pkt->source_win_handle = source_win_handle; + lock_pkt->request_handle = request_handle; lock_pkt->flags = flags; /* replace original pkt with lock pkt */ @@ -456,12 +475,14 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, } issue_ack: - MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); if (pkt->type == MPIDI_CH3_PKT_LOCK) { if (lock_discarded) flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED; else flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED; - mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, flag, source_win_handle); + MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); + MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE((*pkt), request_handle, mpi_errno); + + mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, flag, source_win_handle, request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { @@ -469,7 +490,17 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, else if (data_discarded) flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED; else flag = MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED; - mpi_errno = MPIDI_CH3I_Send_lock_op_ack_pkt(vc, win_ptr, flag, source_win_handle); + if (pkt->type == MPIDI_CH3_PKT_PUT || pkt->type == MPIDI_CH3_PKT_PUT_IMMED || + pkt->type == MPIDI_CH3_PKT_ACCUMULATE || pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { + MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); + request_handle = MPI_REQUEST_NULL; + } + else { + source_win_handle = MPI_WIN_NULL; + MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE((*pkt), request_handle, mpi_errno); + } + + mpi_errno = MPIDI_CH3I_Send_lock_op_ack_pkt(vc, win_ptr, flag, source_win_handle, request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -864,9 +895,11 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, if ((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) pkt_flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; + MPIU_Assert(source_win_handle != MPI_WIN_NULL); mpi_errno = MPIDI_CH3I_Send_lock_op_ack_pkt(vc, win_ptr, pkt_flags, - source_win_handle); + source_win_handle, + MPI_REQUEST_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); MPIDI_CH3_Progress_signal_completion(); } diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 2f3f9ef..d1ae95d 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -240,7 +240,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, } get_accum_resp_pkt->request_handle = rreq->dev.resp_request_handle; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_accum_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) @@ -418,7 +417,6 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *vc, /* Send back data */ MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); fop_resp_pkt->request_handle = rreq->dev.resp_request_handle; - fop_resp_pkt->source_win_handle = rreq->dev.source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -694,13 +692,11 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, sreq->dev.datatype = new_dtp->handle; sreq->dev.datatype_ptr = new_dtp; sreq->dev.target_win_handle = rreq->dev.target_win_handle; - sreq->dev.source_win_handle = rreq->dev.source_win_handle; sreq->dev.flags = rreq->dev.flags; MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); get_resp_pkt->request_handle = rreq->dev.request_handle; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) @@ -1022,7 +1018,6 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_resp_pkt->source_win_handle = get_pkt->source_win_handle; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); @@ -1211,7 +1206,6 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* All origin data is in packet header, issue the header. */ @@ -1270,7 +1264,6 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } fop_resp_pkt->request_handle = fop_pkt->request_handle; - fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1379,7 +1372,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* do final action */ mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, - fop_pkt->flags, fop_pkt->source_win_handle); + fop_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); fn_exit: @@ -1406,7 +1399,6 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; - cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1461,7 +1453,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* do final action */ mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, - cas_pkt->flags, cas_pkt->source_win_handle); + cas_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); fn_exit: @@ -1492,7 +1484,8 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent else { mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(lock_entry->vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, - lock_pkt->source_win_handle); + lock_pkt->source_win_handle, + lock_pkt->request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } } diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c index 93d8e5d..6d0f81d 100644 --- a/src/mpid/ch3/src/ch3u_handle_send_req.c +++ b/src/mpid/ch3/src/ch3u_handle_send_req.c @@ -49,7 +49,6 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = sreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = sreq->dev.flags; /* NOTE: It is possible that this request is already completed before @@ -85,7 +84,7 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; @@ -106,7 +105,6 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc, { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = rreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = rreq->dev.flags; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMSENDCOMPLETE); @@ -150,7 +148,7 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc, on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; @@ -174,7 +172,6 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = rreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = rreq->dev.flags; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_CASSENDCOMPLETE); @@ -218,7 +215,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE/* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; @@ -240,7 +237,6 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc, { int mpi_errno = MPI_SUCCESS; MPID_Win *win_ptr; - MPI_Win source_win_handle = rreq->dev.source_win_handle; MPIDI_CH3_Pkt_flags_t flags = rreq->dev.flags; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPSENDCOMPLETE); @@ -284,7 +280,7 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc, on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - flags, source_win_handle); + flags, MPI_WIN_NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); *complete = TRUE; diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index a43a606..5497210 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -373,7 +373,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype get_pkt->datatype = target_datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; @@ -747,7 +746,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_pkt->datatype = target_datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt == TRUE) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; @@ -838,7 +836,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_accum_pkt->info.dataloop_size = 0; get_accum_pkt->op = op; get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_accum_pkt->source_win_handle = win_ptr->handle; get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { void *src = (void *)origin_addr, *dest = (void *)(get_accum_pkt->info.data); @@ -1090,7 +1087,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, win_ptr->disp_units[target_rank] * target_disp; cas_pkt->datatype = datatype; cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - cas_pkt->source_win_handle = win_ptr->handle; cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; /* REQUIRE: All datatype arguments must be of the same, builtin @@ -1245,7 +1241,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, get_pkt->datatype = datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt == TRUE) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; @@ -1293,7 +1288,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, win_ptr->disp_units[target_rank] * target_disp; fop_pkt->datatype = datatype; fop_pkt->op = op; - fop_pkt->source_win_handle = win_ptr->handle; fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_pkt) { diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 0f4674e..ab77790 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -382,7 +382,6 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req = MPID_Request_create(); req->dev.target_win_handle = get_pkt->target_win_handle; - req->dev.source_win_handle = get_pkt->source_win_handle; req->dev.flags = get_pkt->flags; /* get start location of data and length of data */ @@ -424,7 +423,6 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_resp_pkt->source_win_handle = get_pkt->source_win_handle; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); @@ -764,7 +762,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, resp_req = MPID_Request_create(); resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; - resp_req->dev.source_win_handle = get_accum_pkt->source_win_handle; resp_req->dev.flags = get_accum_pkt->flags; MPIDI_Request_set_type(resp_req, MPIDI_REQUEST_TYPE_GET_ACCUM_RESP); @@ -796,7 +793,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; - get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) @@ -868,7 +864,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.op = get_accum_pkt->op; req->dev.real_user_buf = get_accum_pkt->addr; req->dev.target_win_handle = get_accum_pkt->target_win_handle; - req->dev.source_win_handle = get_accum_pkt->source_win_handle; req->dev.flags = get_accum_pkt->flags; req->dev.resp_request_handle = get_accum_pkt->request_handle; @@ -1024,7 +1019,6 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; - cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1078,7 +1072,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - cas_pkt->flags, cas_pkt->source_win_handle); + cas_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); fn_exit: @@ -1113,7 +1107,8 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_cas_resp); - MPID_Win_get_ptr(cas_resp_pkt->source_win_handle, win_ptr); + MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter on this target */ if (cas_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1130,7 +1125,6 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); MPID_Datatype_get_size_macro(req->dev.datatype, len); MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->info.data, len); @@ -1196,7 +1190,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED); fop_resp_pkt->request_handle = fop_pkt->request_handle; - fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1256,7 +1249,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, - fop_pkt->flags, fop_pkt->source_win_handle); + fop_pkt->flags, MPI_WIN_NULL); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { @@ -1277,7 +1270,6 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.op = fop_pkt->op; req->dev.real_user_buf = fop_pkt->addr; req->dev.target_win_handle = fop_pkt->target_win_handle; - req->dev.source_win_handle = fop_pkt->source_win_handle; req->dev.flags = fop_pkt->flags; req->dev.resp_request_handle = fop_pkt->request_handle; req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPRecvComplete; @@ -1354,9 +1346,8 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_fop_resp); - MPID_Win_get_ptr(fop_resp_pkt->source_win_handle, win_ptr); - MPID_Request_get_ptr(fop_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter */ if (fop_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1440,7 +1431,8 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_accum_resp); - MPID_Win_get_ptr(get_accum_resp_pkt->source_win_handle, win_ptr); + MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter on target */ if (get_accum_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1460,8 +1452,6 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - MPID_Request_get_ptr(get_accum_resp_pkt->request_handle, req); - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; @@ -1538,7 +1528,8 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 1) { /* send lock granted packet. */ mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, - lock_pkt->source_win_handle); + lock_pkt->source_win_handle, + lock_pkt->request_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } @@ -1582,7 +1573,8 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_resp); - MPID_Win_get_ptr(get_resp_pkt->source_win_handle, win_ptr); + MPID_Request_get_ptr(get_resp_pkt->request_handle, req); + MPID_Win_get_ptr(req->dev.source_win_handle, win_ptr); /* decrement ack_counter on target */ if (get_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1602,8 +1594,6 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); - MPID_Request_get_ptr(get_resp_pkt->request_handle, req); - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; @@ -1670,7 +1660,17 @@ int MPIDI_CH3_PktHandler_LockAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); + if (lock_ack_pkt->source_win_handle != MPI_WIN_NULL) { MPID_Win_get_ptr(lock_ack_pkt->source_win_handle, win_ptr); + } + else { + MPIU_Assert(lock_ack_pkt->request_handle != MPI_REQUEST_NULL); + + MPID_Request *req_ptr = NULL; + MPID_Request_get_ptr(lock_ack_pkt->request_handle, req_ptr); + MPIU_Assert(req_ptr->dev.source_win_handle != MPI_REQUEST_NULL); + MPID_Win_get_ptr(req_ptr->dev.source_win_handle, win_ptr); + } mpi_errno = handle_lock_ack(win_ptr, target_rank, lock_ack_pkt->flags); @@ -1705,7 +1705,17 @@ int MPIDI_CH3_PktHandler_LockOpAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); + if (lock_op_ack_pkt->source_win_handle != MPI_WIN_NULL) { MPID_Win_get_ptr(lock_op_ack_pkt->source_win_handle, win_ptr); + } + else { + MPIU_Assert(lock_op_ack_pkt->request_handle != MPI_REQUEST_NULL); + + MPID_Request *req_ptr = NULL; + MPID_Request_get_ptr(lock_op_ack_pkt->request_handle, req_ptr); + MPIU_Assert(req_ptr->dev.source_win_handle != MPI_REQUEST_NULL); + MPID_Win_get_ptr(req_ptr->dev.source_win_handle, win_ptr); + } mpi_errno = handle_lock_ack(win_ptr, target_rank, lock_op_ack_pkt->flags); http://git.mpich.org/mpich.git/commitdiff/42c52518e0179196e8dd8021f6e72cbaa… commit 42c52518e0179196e8dd8021f6e72cbaa9a417fd Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 10:26:45 2015 -0800 Bug-fix: when encounter mpich failure, unlock inter-process lock. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 5943d35..2f3f9ef 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -268,6 +268,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, void *src = (void *)(rreq->dev.real_user_buf), *dest = (void *)(get_accum_resp_pkt->info.data); mpi_errno = immed_copy(src, dest, rreq->dev.user_count * type_size); if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); MPIU_ERR_POP(mpi_errno); } } @@ -279,6 +281,10 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Segment *seg = MPID_Segment_alloc(); MPI_Aint last = type_size * rreq->dev.user_count; + if (seg == NULL) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + } MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment"); MPID_Segment_init(rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, seg, 0); MPID_Segment_pack(seg, 0, &last, resp_req->dev.user_buf); @@ -289,13 +295,12 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, /* accumulate data from tmp_buf into user_buf */ mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumSendComplete; resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumSendComplete; resp_req->dev.target_win_handle = rreq->dev.target_win_handle; @@ -403,14 +408,13 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *vc, if (rreq->dev.op != MPI_NO_OP) { mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, 1, rreq->dev.datatype, rreq->dev.op); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + /* Send back data */ MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); fop_resp_pkt->request_handle = rreq->dev.resp_request_handle; @@ -1074,19 +1078,19 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* All data fits in packet header */ mpi_errno = do_accumulate_op(acc_pkt->info.data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { MPIU_Assert(acc_pkt->type == MPIDI_CH3_PKT_ACCUMULATE); mpi_errno = do_accumulate_op(lock_entry->data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, FALSE /* has no response data */, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1148,7 +1152,11 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { void *src = (void *)(get_accum_pkt->addr), *dest = (void *)(get_accum_resp_pkt->info.data); mpi_errno = immed_copy(src, dest, len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } } else { if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { @@ -1158,6 +1166,10 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_Segment *seg = MPID_Segment_alloc(); MPI_Aint last = type_size * get_accum_pkt->count; + if (seg == NULL) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + } MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment"); MPID_Segment_init(get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, seg, 0); @@ -1170,19 +1182,19 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc /* All data fits in packet header */ mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { MPIU_Assert(get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM); mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + /* here we increment the Active Target counter to guarantee the GET-like operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; @@ -1295,7 +1307,11 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* copy data to resp pkt header */ void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; mpi_errno = immed_copy(src, dest, type_size); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } } else { MPIU_Memcpy(resp_req->dev.user_buf, fop_pkt->addr, @@ -1312,12 +1328,13 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en mpi_errno = do_accumulate_op(lock_entry->data, fop_pkt->addr, 1, fop_pkt->datatype, fop_pkt->op); } - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,lock_entry->vc); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 1421224..0f4674e 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -812,7 +812,11 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* copy data from target buffer to response packet header */ src = (void *) (get_accum_pkt->addr), dest = (void *) (get_accum_resp_pkt->info.data); mpi_errno = immed_copy(src, dest, len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } } else { MPIU_Memcpy(resp_req->dev.user_buf, get_accum_pkt->addr, @@ -823,13 +827,12 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); @@ -1209,20 +1212,23 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* copy data to resp pkt header */ void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; mpi_errno = immed_copy(src, dest, type_size); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + if (mpi_errno != MPI_SUCCESS) { + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + MPIU_ERR_POP(mpi_errno); + } /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { mpi_errno = do_accumulate_op(fop_pkt->info.data, fop_pkt->addr, 1, fop_pkt->datatype, fop_pkt->op); - if (mpi_errno != MPI_SUCCESS) { - MPIU_ERR_POP(mpi_errno); - } } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); http://git.mpich.org/mpich.git/commitdiff/c8ecef8d531601900e9d8db5828489771… commit c8ecef8d531601900e9d8db58284897711295d12 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 10:24:24 2015 -0800 Bug-fix: use do_accumulate_op function for ACC computation. do_accumulate_op() does more comprehensive work on ACC computation than OP function. For example, MPI_REPLACE is not defined as predefined computation and therefore not handled by OP function, but it is safely handled in do_accumulate_op(). This patch replace OP function with do_accumulate_op() on target side. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index c6fc623..5943d35 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -1304,14 +1304,15 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { - MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); - int one = 1; if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { - (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + mpi_errno = do_accumulate_op(fop_pkt->info.data, fop_pkt->addr, + 1, fop_pkt->datatype, fop_pkt->op); } else { - (*uop)(lock_entry->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + mpi_errno = do_accumulate_op(lock_entry->data, fop_pkt->addr, + 1, fop_pkt->datatype, fop_pkt->op); } + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } if (win_ptr->shm_allocated == TRUE) diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 1f70002..1421224 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -1213,9 +1213,11 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { - MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); - int one = 1; - (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + mpi_errno = do_accumulate_op(fop_pkt->info.data, fop_pkt->addr, + 1, fop_pkt->datatype, fop_pkt->op); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } } if (win_ptr->shm_allocated == TRUE) http://git.mpich.org/mpich.git/commitdiff/59afc29cbf253c7cb7f8efb052e816338… commit 59afc29cbf253c7cb7f8efb052e816338cb99304 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Wed Jan 28 10:07:24 2015 -0600 Use memcpy for structure assignment. In this patch we replace "=" with memcpy function when assigning structure content to another struct. Using "=" in this case is not compatible for llvm compiler. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_lockqueue.h b/src/mpid/ch3/include/mpid_rma_lockqueue.h index 1c98ced..13f1290 100644 --- a/src/mpid/ch3/include/mpid_rma_lockqueue.h +++ b/src/mpid/ch3/include/mpid_rma_lockqueue.h @@ -31,7 +31,7 @@ static inline MPIDI_RMA_Lock_entry_t *MPIDI_CH3I_Win_lock_entry_alloc(MPID_Win * if (new_ptr != NULL) { new_ptr->next = NULL; - new_ptr->pkt = (*pkt); + MPIU_Memcpy(&(new_ptr->pkt), pkt, sizeof(*pkt)); new_ptr->vc = NULL; new_ptr->data = NULL; new_ptr->data_size = 0; http://git.mpich.org/mpich.git/commitdiff/1b30ab19a3dfdd6ae31c87dba1928ee68… commit 1b30ab19a3dfdd6ae31c87dba1928ee68ec2775a Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 10:19:40 2015 -0800 Change argument of function finish_op_on_target. In this patch, we replace one argument of function finish_op_on_target, "packet(op) type", with "has_response_data". Since finish_op_on_target does not care what specific packet(op) type it is processing on, but only cares about if the current op has response data (like GET/GACC), changing the argument in this way can simplify the code by avoiding acquiring packet(op) type everytime before calling finish_op_on_target. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index d35e5b2..1b4fdf9 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -851,13 +851,12 @@ static inline int check_piggyback_lock(MPID_Win *win_ptr, MPIDI_VC_t *vc, } static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, - MPIDI_CH3_Pkt_type_t type, + int has_response_data, MPIDI_CH3_Pkt_flags_t flags, MPI_Win source_win_handle) { int mpi_errno = MPI_SUCCESS; - if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_PUT_IMMED || - type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || type == MPIDI_CH3_PKT_ACCUMULATE) { + if (!has_response_data) { /* This is PUT or ACC */ if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index fbb38ab..c6fc623 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -113,7 +113,7 @@ int MPIDI_CH3_ReqHandler_PutRecvComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_PUT, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, flags, source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -191,7 +191,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_ACCUMULATE, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, flags, source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -957,7 +957,7 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, put_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, FALSE /* has no response data */, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1087,7 +1087,7 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, acc_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, FALSE /* has no response data */, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1360,7 +1360,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, fop_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1442,7 +1442,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, cas_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, TRUE /* has response data */, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c index 84ad691..93d8e5d 100644 --- a/src/mpid/ch3/src/ch3u_handle_send_req.c +++ b/src/mpid/ch3/src/ch3u_handle_send_req.c @@ -84,7 +84,7 @@ int MPIDI_CH3_ReqHandler_GetSendComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_GET, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -149,7 +149,7 @@ int MPIDI_CH3_ReqHandler_GaccumSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_GET_ACCUM, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -217,7 +217,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS_IMMED, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE/* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -283,7 +283,7 @@ int MPIDI_CH3_ReqHandler_FOPSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 4163baa..1f70002 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -225,7 +225,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_Memcpy(put_pkt->addr, put_pkt->info.data, put_pkt->count*type_size); /* trigger final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -577,7 +577,7 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } /* trigger final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, FALSE /* has no response data */, accum_pkt->flags, accum_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1074,7 +1074,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Request_release(req); } - mpi_errno = finish_op_on_target(win_ptr, vc, cas_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1247,7 +1247,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } } - mpi_errno = finish_op_on_target(win_ptr, vc, fop_pkt->type, + mpi_errno = finish_op_on_target(win_ptr, vc, TRUE /* has response data */, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } http://git.mpich.org/mpich.git/commitdiff/21479b00b1080d577be8ff7ed66a7bdd6… commit 21479b00b1080d577be8ff7ed66a7bdd62dd16ac Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Mon Feb 9 22:35:28 2015 -0800 Add asserts for RMA packet types. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index ea1036d..fbb38ab 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -936,6 +936,12 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_put_t *put_pkt = &((lock_entry->pkt).put); int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + if (put_pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { /* all data fits in packet header */ mpi_errno = MPIR_Localcopy(put_pkt->info.data, put_pkt->count, put_pkt->datatype, @@ -973,6 +979,12 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + sreq = MPID_Request_create(); if (sreq == NULL) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); @@ -1052,6 +1064,9 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIU_Assert(lock_entry->all_data_recved == 1); + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(acc_pkt->datatype)); + if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); @@ -1095,6 +1110,12 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + sreq = MPID_Request_create(); if (sreq == NULL) { MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); @@ -1218,6 +1239,12 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en int iovcnt; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(fop_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + /* FIXME: this function is same with PktHandler_FOP(), should do code refactoring on both of them. */ @@ -1353,6 +1380,12 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPI_Aint len; int mpi_errno = MPI_SUCCESS; + /* Piggyback candidate should have basic datatype for target datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(cas_pkt->datatype)); + + /* Make sure that all data is received for this op. */ + MPIU_Assert(lock_entry->all_data_recved == 1); + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; http://git.mpich.org/mpich.git/commitdiff/de9d0f21b33691ee555d3479acdc5e586… commit de9d0f21b33691ee555d3479acdc5e5863af92ca Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Mon Feb 9 19:14:32 2015 -0800 Rewrite code of piggybacking IMMED data with RMA packets. Originally we add "immed_data" and "immed_len" areas to RMA packets, in order to piggyback small amount of data with packet header to reduce number of packets (Note that "immed_len" is necessary when the piggybacked data is not the entire data). However, those areas potentially increase the packet union size and worsen the two-sided communication. This patch fixes this issue. In this patch, we remove "immed_data" and "immed_len" from normal "MPIDI_CH3_Pkt_XXX_t" operation type (e.g. MPIDI_CH3_Pkt_put_t), and we introduce new "MPIDI_CH3_Pkt_XXX_immed_t" packt type for each operation (e.g. MPIDI_CH3_Pkt_put_immed_t). "MPIDI_CH3_Pkt_XXX_immed_t" is used when (1) both origin and target are basic datatypes, AND, (2) the data to be sent can be entirely fit into the header. By doing this, "MPIDI_CH3_Pkt_XXX_immed_t" needs "immed_data" area but can drop "immed_len" area. Also, since it only works with basic target datatype, it can drop "dataloop_size" area as well. All operations that do not satisfy (1) or (2) will use normal "MPIDI_CH3_Pkt_XXX_t" type. Originally we always piggyback FOP data into the packet header, which makes the packet size too large. In this patch we split the FOP operaton into IMMED packets and normal packets. Because CAS only work with 2 basic datatype and non-complex elements, the data amount is relatively small, we always piggyback the data with packet header and only use "MPIDI_CH3_Pkt_XXX_immed_t" packet type for CAS. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index a443521..2c1a821 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -326,7 +326,6 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags) { MPIDI_VC_t *vc = NULL; - size_t len; MPI_Aint origin_type_size; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_put_t *put_pkt = &rma_op->pkt.put; @@ -341,23 +340,10 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, put_pkt->flags |= flags; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - if (!rma_op->is_dt) { - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(put_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (put_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = put_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)put_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (len == (size_t)put_pkt->immed_len) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, put_pkt, sizeof(*put_pkt), &(rma_op->request)); @@ -368,10 +354,8 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, /* We still need to issue from origin buffer. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) put_pkt; iov[0].MPID_IOV_LEN = sizeof(*put_pkt); - if (!rma_op->is_dt) { - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr + put_pkt->immed_len); - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size - put_pkt->immed_len; - } + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -397,7 +381,6 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags) { MPIDI_VC_t *vc = NULL; - size_t len; MPI_Aint origin_type_size; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_accum_t *accum_pkt = &rma_op->pkt.accum; @@ -412,23 +395,10 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, accum_pkt->flags |= flags; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - if (!rma_op->is_dt) { - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(accum_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (accum_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = accum_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)accum_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (len == (size_t)accum_pkt->immed_len) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, accum_pkt, sizeof(*accum_pkt), &(rma_op->request)); @@ -439,10 +409,8 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, /* We still need to issue from origin buffer. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*accum_pkt); - if (!rma_op->is_dt) { - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr + accum_pkt->immed_len); - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size - accum_pkt->immed_len; - } + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -466,7 +434,6 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, MPIDI_CH3_Pkt_flags_t flags) { MPIDI_VC_t *vc = NULL; - size_t len; MPI_Aint origin_type_size; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt = &rma_op->pkt.get_accum; @@ -505,30 +472,12 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, get_accum_pkt->request_handle = resp_req->handle; get_accum_pkt->flags |= flags; - if (!rma_op->is_dt) { - /* Only fill IMMED data in response packet when both origin and target - buffers are basic datatype. */ - get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; - } MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - if (!rma_op->is_dt) { - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(get_accum_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (get_accum_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = get_accum_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)get_accum_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); - if (len == (size_t)get_accum_pkt->immed_len) { + if (rma_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, get_accum_pkt, sizeof(*get_accum_pkt), &(rma_op->request)); @@ -539,10 +488,8 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, /* We still need to issue from origin buffer. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_pkt); - if (!rma_op->is_dt) { - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr + get_accum_pkt->immed_len); - iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size - get_accum_pkt->immed_len; - } + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = rma_op->origin_count * origin_type_size; mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -636,11 +583,6 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, get_pkt->request_handle = rma_op->request->handle; get_pkt->flags |= flags; - if (!rma_op->is_dt) { - /* Only fill IMMED data in response packet when both origin and target - buffers are basic datatype. */ - get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; - } comm_ptr = win_ptr->comm_ptr; MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); @@ -705,7 +647,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, MPIDI_RMA_Target_t *target_ptr, MPIDI_CH3_Pkt_flags_t flags) { - MPI_Aint len; MPIDI_VC_t *vc = NULL; MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_cas_t *cas_pkt = &rma_op->pkt.cas; @@ -729,20 +670,12 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, rma_op->request->dev.user_count = rma_op->result_count; rma_op->request->dev.datatype = rma_op->result_datatype; - /* REQUIRE: All datatype arguments must be of the same, builtin - * type and counts must be 1. */ - MPID_Datatype_get_size_macro(rma_op->origin_datatype, len); - MPIU_Assert(len <= sizeof(MPIDI_CH3_CAS_Immed_u)); - rma_op->request->dev.target_win_handle = cas_pkt->target_win_handle; rma_op->request->dev.source_win_handle = cas_pkt->source_win_handle; cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags |= flags; - MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); - MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_pkt, sizeof(*cas_pkt), &rmw_req); @@ -782,8 +715,8 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, MPID_Comm *comm_ptr = win_ptr->comm_ptr; MPIDI_CH3_Pkt_fop_t *fop_pkt = &rma_op->pkt.fop; MPID_Request *resp_req = NULL; - size_t len; MPI_Aint origin_type_size; + MPID_IOV iov[MPID_IOV_LIMIT]; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_ISSUE_FOP_OP); @@ -809,26 +742,27 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, fop_pkt->flags |= flags; - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); - - /* Fill origin data into packet header IMMED area as much as possible */ - MPIU_Assign_trunc(fop_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size), - int); - if (fop_pkt->immed_len > 0) { - void *src = rma_op->origin_addr, *dest = fop_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)fop_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); + MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); + + if (rma_op->pkt.type == MPIDI_CH3_PKT_FOP_IMMED) { /* All origin data is in packet header, issue the header. */ MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_pkt, sizeof(*fop_pkt), &(rma_op->request)); MPIU_THREAD_CS_EXIT(CH3COMM, vc); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + else { + /* We still need to issue from origin buffer. */ + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)rma_op->origin_addr); + iov[1].MPID_IOV_LEN = origin_type_size; + + mpi_errno = issue_from_origin_buffer(rma_op, iov, vc); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } /* This operation can generate two requests; one for inbound and one for * outbound data. */ @@ -888,21 +822,25 @@ static inline int issue_rma_op(MPIDI_RMA_Op_t * op_ptr, MPID_Win * win_ptr, switch (op_ptr->pkt.type) { case (MPIDI_CH3_PKT_PUT): + case (MPIDI_CH3_PKT_PUT_IMMED): mpi_errno = issue_put_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_ACCUMULATE): + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): mpi_errno = issue_acc_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_GET_ACCUM): + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): mpi_errno = issue_get_acc_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_GET): mpi_errno = issue_get_op(op_ptr, win_ptr, target_ptr, flags); break; - case (MPIDI_CH3_PKT_CAS): + case (MPIDI_CH3_PKT_CAS_IMMED): mpi_errno = issue_cas_op(op_ptr, win_ptr, target_ptr, flags); break; case (MPIDI_CH3_PKT_FOP): + case (MPIDI_CH3_PKT_FOP_IMMED): mpi_errno = issue_fop_op(op_ptr, win_ptr, target_ptr, flags); break; default: diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 1c924be..6b02880 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1913,6 +1913,8 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *, MPID_Request *, int * ); int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *, MPID_Request *, int * ); +int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *, MPID_Request *, + int * ); int MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete( MPIDI_VC_t *, MPID_Request *, int * ); diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index f6e5327..cbd300c 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -76,15 +76,22 @@ typedef enum { MPIDI_CH3_PKT_CANCEL_SEND_RESP, /* RMA Packets begin here */ MPIDI_CH3_PKT_PUT, + MPIDI_CH3_PKT_PUT_IMMED, MPIDI_CH3_PKT_GET, MPIDI_CH3_PKT_ACCUMULATE, + MPIDI_CH3_PKT_ACCUMULATE_IMMED, MPIDI_CH3_PKT_GET_ACCUM, + MPIDI_CH3_PKT_GET_ACCUM_IMMED, MPIDI_CH3_PKT_FOP, - MPIDI_CH3_PKT_CAS, + MPIDI_CH3_PKT_FOP_IMMED, + MPIDI_CH3_PKT_CAS_IMMED, MPIDI_CH3_PKT_GET_RESP, + MPIDI_CH3_PKT_GET_RESP_IMMED, MPIDI_CH3_PKT_GET_ACCUM_RESP, + MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED, MPIDI_CH3_PKT_FOP_RESP, - MPIDI_CH3_PKT_CAS_RESP, + MPIDI_CH3_PKT_FOP_RESP_IMMED, + MPIDI_CH3_PKT_CAS_RESP_IMMED, MPIDI_CH3_PKT_LOCK, MPIDI_CH3_PKT_LOCK_ACK, MPIDI_CH3_PKT_LOCK_OP_ACK, @@ -198,21 +205,25 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ datatype_ = (pkt_).put.datatype; \ break; \ case (MPIDI_CH3_PKT_GET): \ datatype_ = (pkt_).get.datatype; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ datatype_ = (pkt_).accum.datatype; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ datatype_ = (pkt_).get_accum.datatype; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ datatype_ = (pkt_).cas.datatype; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ datatype_ = (pkt_).fop.datatype; \ break; \ default: \ @@ -227,19 +238,23 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ count_ = (pkt_).put.count; \ break; \ case (MPIDI_CH3_PKT_GET): \ count_ = (pkt_).get.count; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ count_ = (pkt_).accum.count; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ count_ = (pkt_).get_accum.count; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ count_ = 1; \ break; \ default: \ @@ -247,47 +262,6 @@ MPIDI_CH3_PKT_DEFS } \ } -#define MPIDI_CH3_PKT_RMA_GET_IMMED_LEN(pkt_, immed_len_, err_) \ - { \ - /* This macro returns immed_len in RMA operation \ - packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ - packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ - err_ = MPI_SUCCESS; \ - switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - immed_len_ = (pkt_).put.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - immed_len_ = (pkt_).accum.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - immed_len_ = (pkt_).get_accum.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - immed_len_ = (pkt_).fop.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_CAS): \ - /* Note that here we return size of origin data, not \ - size of compare data. */ \ - immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ - break; \ - case (MPIDI_CH3_PKT_GET_RESP): \ - immed_len_ = (pkt_).get_resp.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ - immed_len_ = (pkt_).get_accum_resp.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_FOP_RESP): \ - immed_len_ = (pkt_).fop_resp.immed_len; \ - break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ - immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ - break; \ - default: \ - MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ - } \ - } - #define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_) \ { \ /* This macro returns pointer to immed data in RMA operation \ @@ -295,34 +269,34 @@ MPIDI_CH3_PKT_DEFS packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - immed_data_ = (pkt_).put.data; \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ + immed_data_ = (pkt_).put.info.data; \ break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - immed_data_ = (pkt_).accum.data; \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ + immed_data_ = (pkt_).accum.info.data; \ break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - immed_data_ = (pkt_).get_accum.data; \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ + immed_data_ = (pkt_).get_accum.info.data; \ break; \ - case (MPIDI_CH3_PKT_FOP): \ - immed_data_ = (pkt_).fop.data; \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ + immed_data_ = (pkt_).fop.info.data; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ /* Note that here we return pointer of origin data, not \ pointer of compare data. */ \ immed_data_ = &((pkt_).cas.origin_data); \ break; \ - case (MPIDI_CH3_PKT_GET_RESP): \ - immed_data_ = (pkt_).get_resp.data; \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ + immed_data_ = (pkt_).get_resp.info.data; \ break; \ - case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ - immed_data_ = (pkt_).get_accum_resp.data; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ + immed_data_ = (pkt_).get_accum_resp.info.data; \ break; \ - case (MPIDI_CH3_PKT_FOP_RESP): \ - immed_data_ = (pkt_).fop_resp.data; \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ + immed_data_ = (pkt_).fop_resp.info.data; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ - immed_data_ = &((pkt_).cas_resp.data); \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ + immed_data_ = &((pkt_).cas_resp.info.data); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ @@ -339,33 +313,40 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ flags_ = (pkt_).put.flags; \ break; \ case (MPIDI_CH3_PKT_GET): \ flags_ = (pkt_).get.flags; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ flags_ = (pkt_).accum.flags; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ flags_ = (pkt_).get_accum.flags; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ flags_ = (pkt_).cas.flags; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ flags_ = (pkt_).fop.flags; \ break; \ case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ flags_ = (pkt_).get_resp.flags; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ flags_ = (pkt_).get_accum_resp.flags; \ break; \ case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ flags_ = (pkt_).fop_resp.flags; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ flags_ = (pkt_).cas_resp.flags; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -395,33 +376,40 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET): \ (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -451,33 +439,40 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ win_hdl_ = (pkt_).put.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET): \ win_hdl_ = (pkt_).get.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ win_hdl_ = (pkt_).accum.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ win_hdl_ = (pkt_).get_accum.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ win_hdl_ = (pkt_).cas.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ win_hdl_ = (pkt_).fop.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_RESP): \ + case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ win_hdl_ = (pkt_).get_resp.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ win_hdl_ = (pkt_).get_accum_resp.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_FOP_RESP): \ + case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ win_hdl_ = (pkt_).fop_resp.source_win_handle; \ break; \ - case (MPIDI_CH3_PKT_CAS_RESP): \ + case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ win_hdl_ = (pkt_).cas_resp.source_win_handle; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -511,21 +506,25 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ + case (MPIDI_CH3_PKT_PUT_IMMED): \ win_hdl_ = (pkt_).put.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET): \ win_hdl_ = (pkt_).get.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ win_hdl_ = (pkt_).accum.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ win_hdl_ = (pkt_).get_accum.target_win_handle; \ break; \ - case (MPIDI_CH3_PKT_CAS): \ + case (MPIDI_CH3_PKT_CAS_IMMED): \ win_hdl_ = (pkt_).cas.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_FOP): \ + case (MPIDI_CH3_PKT_FOP_IMMED): \ win_hdl_ = (pkt_).fop.target_win_handle; \ break; \ case (MPIDI_CH3_PKT_LOCK): \ @@ -552,30 +551,28 @@ MPIDI_CH3_PKT_DEFS err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ - (pkt_).put.dataloop_size = (dataloop_size_); \ + (pkt_).put.info.dataloop_size = (dataloop_size_); \ break; \ case (MPIDI_CH3_PKT_GET): \ - (pkt_).get.dataloop_size = (dataloop_size_); \ + (pkt_).get.info.dataloop_size = (dataloop_size_); \ break; \ case (MPIDI_CH3_PKT_ACCUMULATE): \ - (pkt_).accum.dataloop_size = (dataloop_size_); \ + (pkt_).accum.info.dataloop_size = (dataloop_size_); \ break; \ case (MPIDI_CH3_PKT_GET_ACCUM): \ - (pkt_).get_accum.dataloop_size = (dataloop_size_); \ + (pkt_).get_accum.info.dataloop_size = (dataloop_size_); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ } - typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in * active target rma and for unlocking window @@ -583,8 +580,10 @@ typedef struct MPIDI_CH3_Pkt_put { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + union { + int dataloop_size; + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_put_t; typedef struct MPIDI_CH3_Pkt_get { @@ -593,7 +592,12 @@ typedef struct MPIDI_CH3_Pkt_get { void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ + struct { + /* note that we use struct here in order + to consistently access dataloop_size + by "pkt->info.dataloop_size". */ + int dataloop_size; /* for derived datatypes */ + } info; MPI_Request request_handle; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in @@ -612,8 +616,12 @@ typedef struct MPIDI_CH3_Pkt_get_resp { MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_get_resp_t; typedef struct MPIDI_CH3_Pkt_accum { @@ -622,7 +630,6 @@ typedef struct MPIDI_CH3_Pkt_accum { void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ MPI_Op op; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in @@ -631,8 +638,10 @@ typedef struct MPIDI_CH3_Pkt_accum { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + union { + int dataloop_size; + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum { @@ -642,7 +651,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum { void *addr; int count; MPI_Datatype datatype; - int dataloop_size; /* for derived datatypes */ MPI_Op op; MPI_Win target_win_handle; /* Used in the last RMA operation in each * epoch for decrementing rma op counter in @@ -651,8 +659,10 @@ typedef struct MPIDI_CH3_Pkt_get_accum { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + union { + int dataloop_size; + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_get_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum_resp { @@ -663,8 +673,12 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_cas { @@ -685,7 +699,12 @@ typedef struct MPIDI_CH3_Pkt_cas { typedef struct MPIDI_CH3_Pkt_cas_resp { MPIDI_CH3_Pkt_type_t type; MPI_Request request_handle; - MPIDI_CH3_CAS_Immed_u data; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + MPIDI_CH3_CAS_Immed_u data; + } info; /* followings are used to decrement ack_counter at orign */ int target_rank; MPI_Win source_win_handle; @@ -704,15 +723,23 @@ typedef struct MPIDI_CH3_Pkt_fop { * epoch for decrementing rma op counter in * active target rma and for unlocking window * in passive target rma. Otherwise set to NULL*/ - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; } MPIDI_CH3_Pkt_fop_t; typedef struct MPIDI_CH3_Pkt_fop_resp { MPIDI_CH3_Pkt_type_t type; MPI_Request request_handle; - char data[MPIDI_RMA_IMMED_BYTES]; - int immed_len; + struct { + /* note that we use struct here in order + to consistently access data + by "pkt->info.data". */ + char data[MPIDI_RMA_IMMED_BYTES]; + } info; /* followings are used to decrement ack_counter at orign */ int target_rank; MPI_Win source_win_handle; diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 2bb62b2..9c2618d 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -433,7 +433,6 @@ typedef struct MPIDI_Request { MPI_Op op; /* For accumulate, since data is first read into a tmp_buf */ void *real_user_buf; - void *final_user_buf; /* For derived datatypes at target */ struct MPIDI_RMA_dtype_info *dtype_info; void *dataloop; diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index d7a720d..d35e5b2 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -333,9 +333,12 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, } if (pkt->type == MPIDI_CH3_PKT_LOCK || + pkt->type == MPIDI_CH3_PKT_PUT_IMMED || + pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || pkt->type == MPIDI_CH3_PKT_GET || - pkt->type == MPIDI_CH3_PKT_FOP || - pkt->type == MPIDI_CH3_PKT_CAS) { + pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || + pkt->type == MPIDI_CH3_PKT_FOP_IMMED || + pkt->type == MPIDI_CH3_PKT_CAS_IMMED) { /* return bytes of data processed in this pkt handler */ (*buflen) = sizeof(MPIDI_CH3_Pkt_t); @@ -351,8 +354,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPID_Request *req = NULL; MPI_Datatype target_dtp; int target_count; - int immed_len = 0; - void *immed_data = NULL; int complete = 0; MPIDI_msg_sz_t data_len; char *data_buf = NULL; @@ -365,17 +366,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPID_Datatype_get_size_macro(target_dtp, type_size); recv_data_sz = type_size * target_count; - if (recv_data_sz <= MPIDI_RMA_IMMED_BYTES) { - - /* return bytes of data processed in this pkt handler */ - (*buflen) = sizeof(MPIDI_CH3_Pkt_t); - - if (new_ptr != NULL) - new_ptr->all_data_recved = 1; - - goto issue_ack; - } - if (new_ptr != NULL) { if (win_ptr->current_lock_data_bytes + recv_data_sz < MPIR_CVAR_CH3_RMA_LOCK_DATA_BYTES) { @@ -430,10 +420,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, req->dev.OnFinal = MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete; req->dev.lock_queue_entry = new_ptr; - MPIDI_CH3_PKT_RMA_GET_IMMED_LEN((*pkt), immed_len, mpi_errno); - if (immed_len > 0) { - req->dev.recv_data_sz -= immed_len; - } data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); MPIU_Assert(req->dev.recv_data_sz > 0); @@ -447,15 +433,6 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, req->dev.OnFinal = MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete; req->dev.lock_queue_entry = new_ptr; - MPIDI_CH3_PKT_RMA_GET_IMMED_LEN((*pkt), immed_len, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR((*pkt), immed_data, mpi_errno); - - if (immed_len > 0) { - /* see if we can receive some data from packet header */ - MPIU_Memcpy(req->dev.user_buf, immed_data, (size_t)immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + immed_len); - req->dev.recv_data_sz -= immed_len; - } data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); MPIU_Assert(req->dev.recv_data_sz > 0); @@ -597,7 +574,9 @@ static inline int adjust_op_piggybacked_with_lock (MPID_Win *win_ptr, &(target->dt_op_list_tail), op); } else if (op->pkt.type == MPIDI_CH3_PKT_PUT || - op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || + op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || + op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list), &(target->write_op_list_tail), op); } @@ -877,7 +856,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPI_Win source_win_handle) { int mpi_errno = MPI_SUCCESS; - if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_ACCUMULATE) { + if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_PUT_IMMED || + type == MPIDI_CH3_PKT_ACCUMULATE_IMMED || type == MPIDI_CH3_PKT_ACCUMULATE) { /* This is PUT or ACC */ if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { diff --git a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c index c2ae4a7..919d097 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c @@ -573,12 +573,18 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[], We could even do lazy initialization (make this part of win_create) */ pktArray[MPIDI_CH3_PKT_PUT] = MPIDI_CH3_PktHandler_Put; + pktArray[MPIDI_CH3_PKT_PUT_IMMED] = + MPIDI_CH3_PktHandler_Put; pktArray[MPIDI_CH3_PKT_ACCUMULATE] = MPIDI_CH3_PktHandler_Accumulate; + pktArray[MPIDI_CH3_PKT_ACCUMULATE_IMMED] = + MPIDI_CH3_PktHandler_Accumulate; pktArray[MPIDI_CH3_PKT_GET] = MPIDI_CH3_PktHandler_Get; pktArray[MPIDI_CH3_PKT_GET_RESP] = MPIDI_CH3_PktHandler_GetResp; + pktArray[MPIDI_CH3_PKT_GET_RESP_IMMED] = + MPIDI_CH3_PktHandler_GetResp; pktArray[MPIDI_CH3_PKT_LOCK] = MPIDI_CH3_PktHandler_Lock; pktArray[MPIDI_CH3_PKT_LOCK_ACK] = @@ -593,18 +599,26 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[], MPIDI_CH3_PktHandler_FlushAck; pktArray[MPIDI_CH3_PKT_DECR_AT_COUNTER] = MPIDI_CH3_PktHandler_DecrAtCnt; - pktArray[MPIDI_CH3_PKT_CAS] = + pktArray[MPIDI_CH3_PKT_CAS_IMMED] = MPIDI_CH3_PktHandler_CAS; - pktArray[MPIDI_CH3_PKT_CAS_RESP] = + pktArray[MPIDI_CH3_PKT_CAS_RESP_IMMED] = MPIDI_CH3_PktHandler_CASResp; pktArray[MPIDI_CH3_PKT_FOP] = MPIDI_CH3_PktHandler_FOP; + pktArray[MPIDI_CH3_PKT_FOP_IMMED] = + MPIDI_CH3_PktHandler_FOP; pktArray[MPIDI_CH3_PKT_FOP_RESP] = MPIDI_CH3_PktHandler_FOPResp; + pktArray[MPIDI_CH3_PKT_FOP_RESP_IMMED] = + MPIDI_CH3_PktHandler_FOPResp; pktArray[MPIDI_CH3_PKT_GET_ACCUM] = MPIDI_CH3_PktHandler_GetAccumulate; + pktArray[MPIDI_CH3_PKT_GET_ACCUM_IMMED] = + MPIDI_CH3_PktHandler_GetAccumulate; pktArray[MPIDI_CH3_PKT_GET_ACCUM_RESP] = MPIDI_CH3_PktHandler_Get_AccumResp; + pktArray[MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED] = + MPIDI_CH3_PktHandler_Get_AccumResp; /* End of default RMA operations */ /* Fault tolerance */ diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 302e167..ea1036d 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -172,7 +172,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc, if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); /* accumulate data from tmp_buf into user_buf */ - mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf, + mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); @@ -182,7 +182,7 @@ int MPIDI_CH3_ReqHandler_AccumRecvComplete( MPIDI_VC_t *vc, /* free the temporary buffer */ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); - MPIU_Free((char *) rreq->dev.final_user_buf + true_lb); + MPIU_Free((char *) rreq->dev.user_buf + true_lb); /* mark data transfer as complete and decrement CC */ MPIDI_CH3U_Request_complete(rreq); @@ -224,7 +224,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Request *resp_req; MPID_IOV iov[MPID_IOV_LIMIT]; MPI_Aint true_lb, true_extent; - size_t len; int iovcnt; MPIU_CHKPMEM_DECL(1); MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE); @@ -233,7 +232,12 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); + if ((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); + } + else { MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); + } get_accum_resp_pkt->request_handle = rreq->dev.resp_request_handle; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_accum_resp_pkt->source_win_handle = rreq->dev.source_win_handle; @@ -244,7 +248,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - get_accum_resp_pkt->immed_len = 0; MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size); @@ -253,12 +256,22 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); MPIU_Object_set_ref(resp_req, 1); + if (!((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, rreq->dev.user_count * type_size, mpi_errno, "GACC resp. buffer"); + } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if ((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + void *src = (void *)(rreq->dev.real_user_buf), *dest = (void *)(get_accum_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, rreq->dev.user_count * type_size); + if (mpi_errno != MPI_SUCCESS) { + MPIU_ERR_POP(mpi_errno); + } + } + else { if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) { MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count * type_size); @@ -271,9 +284,10 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Segment_pack(seg, 0, &last, resp_req->dev.user_buf); MPID_Segment_free(seg); } + } /* accumulate data from tmp_buf into user_buf */ - mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf, + mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -291,27 +305,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; - /* length of target data */ - MPIU_Assign_trunc(len, rreq->dev.user_count * type_size, size_t); - - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ - if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_accum_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_accum_resp_pkt->immed_len > 0) { - void *src = resp_req->dev.user_buf; - void *dest = (void*) get_accum_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_accum_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } - - if (len == (size_t)get_accum_resp_pkt->immed_len) { + if ((rreq->dev.flags) & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); @@ -320,8 +314,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf + get_accum_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = rreq->dev.user_count * type_size - get_accum_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = rreq->dev.user_count * type_size; iovcnt = 2; } @@ -338,7 +332,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, /* free the temporary buffer */ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); - MPIU_Free((char *) rreq->dev.final_user_buf + true_lb); + MPIU_Free((char *) rreq->dev.user_buf + true_lb); /* mark data transfer as complete and decrement CC */ MPIDI_CH3U_Request_complete(rreq); @@ -355,6 +349,112 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, /* --END ERROR HANDLING-- */ } + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_ReqHandler_FOPRecvComplete +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPIDI_CH3_ReqHandler_FOPRecvComplete( MPIDI_VC_t *vc, + MPID_Request *rreq, + int *complete ) +{ + int mpi_errno = MPI_SUCCESS; + MPID_Win *win_ptr = NULL; + MPI_Aint type_size; + MPID_Request *resp_req = NULL; + MPID_IOV iov[MPID_IOV_LIMIT]; + int iovcnt; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; + MPIU_CHKPMEM_DECL(1); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + + MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + + MPIU_Assert(!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + + MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); + + MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size); + + /* Create response request */ + resp_req = MPID_Request_create(); + MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + MPIU_Object_set_ref(resp_req, 1); + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_FOPSendComplete; + resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPSendComplete; + resp_req->dev.target_win_handle = rreq->dev.target_win_handle; + resp_req->dev.flags = rreq->dev.flags; + + MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, type_size, + mpi_errno, "FOP resp. buffer"); + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + + /* Copy data into a temporary buffer in response request */ + MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, type_size); + + /* Perform accumulate computation */ + if (rreq->dev.op != MPI_NO_OP) { + mpi_errno = do_accumulate_op(rreq->dev.user_buf, rreq->dev.real_user_buf, + 1, rreq->dev.datatype, rreq->dev.op); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + + /* Send back data */ + MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); + fop_resp_pkt->request_handle = rreq->dev.resp_request_handle; + fop_resp_pkt->source_win_handle = rreq->dev.source_win_handle; + fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; + fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) + fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; + if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || + (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) + fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; + + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_resp_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = type_size; + iovcnt = 2; + + MPIU_THREAD_CS_ENTER(CH3COMM,vc); + mpi_errno = MPIDI_CH3_iSendv(vc, resp_req, iov, iovcnt); + MPIU_THREAD_CS_EXIT(CH3COMM,vc); + + MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + + /* free the temporary buffer */ + MPIU_Free((char *) rreq->dev.user_buf); + + /* mark data transfer as complete and decrement CC */ + MPIDI_CH3U_Request_complete(rreq); + *complete = TRUE; + + fn_exit: + MPIU_CHKPMEM_COMMIT(); + MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE); + return MPI_SUCCESS; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + + #undef FUNCNAME #define FUNCNAME MPIDI_CH3_ReqHandler_PutDerivedDTRecvComplete #undef FCNAME @@ -444,7 +544,6 @@ int MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete( MPIDI_VC_t *vc ATTRIBUTE((u tmp_buf = (void *)((char*)tmp_buf - true_lb); rreq->dev.user_buf = tmp_buf; - rreq->dev.final_user_buf = rreq->dev.user_buf; rreq->dev.datatype = new_dtp->handle; rreq->dev.recv_data_sz = new_dtp->size * rreq->dev.user_count; @@ -494,6 +593,8 @@ int MPIDI_CH3_ReqHandler_GaccumDerivedDTRecvComplete( MPIDI_VC_t *vc ATTRIBUTE(( MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMDERIVEDDTRECVCOMPLETE); + MPIU_Assert(!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + /* create derived datatype */ create_derived_datatype(rreq, &new_dtp); @@ -516,7 +617,6 @@ int MPIDI_CH3_ReqHandler_GaccumDerivedDTRecvComplete( MPIDI_VC_t *vc ATTRIBUTE(( tmp_buf = (void *)((char*)tmp_buf - true_lb); rreq->dev.user_buf = tmp_buf; - rreq->dev.final_user_buf = rreq->dev.user_buf; rreq->dev.datatype = new_dtp->handle; rreq->dev.recv_data_sz = new_dtp->size * rreq->dev.user_count; @@ -571,6 +671,8 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr); + MPIU_Assert(!(rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + /* create derived datatype */ create_derived_datatype(rreq, &new_dtp); MPIU_Free(rreq->dev.dtype_info); @@ -602,7 +704,6 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - get_resp_pkt->immed_len = 0; sreq->dev.segment_ptr = MPID_Segment_alloc( ); MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc"); @@ -835,20 +936,22 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_put_t *put_pkt = &((lock_entry->pkt).put); int mpi_errno = MPI_SUCCESS; - if (lock_entry->data == NULL) { + if (put_pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { /* all data fits in packet header */ - mpi_errno = MPIR_Localcopy(put_pkt->data, put_pkt->count, put_pkt->datatype, + mpi_errno = MPIR_Localcopy(put_pkt->info.data, put_pkt->count, put_pkt->datatype, put_pkt->addr, put_pkt->count, put_pkt->datatype); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { + MPIU_Assert(put_pkt->type == MPIDI_CH3_PKT_PUT); + mpi_errno = MPIR_Localcopy(lock_entry->data, put_pkt->count, put_pkt->datatype, put_pkt->addr, put_pkt->count, put_pkt->datatype); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_PUT, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, put_pkt->type, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -888,7 +991,12 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP_IMMED); + } + else { MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + } get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -899,30 +1007,18 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_resp_pkt->source_win_handle = get_pkt->source_win_handle; - get_resp_pkt->immed_len = 0; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t); - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_resp_pkt->immed_len > 0) { - void *src = get_pkt->addr; - void *dest = (void*) get_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } + void *src = (void *)(get_pkt->addr), *dest = (void *)(get_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - if (len == (size_t)get_resp_pkt->immed_len) { + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); @@ -931,8 +1027,8 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)get_pkt->addr + get_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = get_pkt->count * type_size - get_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) (get_pkt->addr); + iov[1].MPID_IOV_LEN = get_pkt->count * type_size; iovcnt = 2; } @@ -959,13 +1055,15 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - if (lock_entry->data == NULL) { + if (acc_pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { /* All data fits in packet header */ - mpi_errno = do_accumulate_op(acc_pkt->data, acc_pkt->addr, + mpi_errno = do_accumulate_op(acc_pkt->info.data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { + MPIU_Assert(acc_pkt->type == MPIDI_CH3_PKT_ACCUMULATE); + mpi_errno = do_accumulate_op(lock_entry->data, acc_pkt->addr, acc_pkt->count, acc_pkt->datatype, acc_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -974,7 +1072,7 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_ACCUMULATE, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, acc_pkt->type, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1013,12 +1111,25 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc /* Copy data into a temporary buffer */ MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); + if (!(get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) sreq->dev.user_buf = (void *)MPIU_Malloc(get_accum_pkt->count * type_size); + else { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); + } + + /* length of target data */ + MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); /* Perform ACCUMULATE OP */ if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + void *src = (void *)(get_accum_pkt->addr), *dest = (void *)(get_accum_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { MPIU_Memcpy(sreq->dev.user_buf, get_accum_pkt->addr, get_accum_pkt->count * type_size); @@ -1032,14 +1143,17 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_Segment_pack(seg, 0, &last, sreq->dev.user_buf); MPID_Segment_free(seg); } + } - if (lock_entry->data == NULL) { + if (get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { /* All data fits in packet header */ - mpi_errno = do_accumulate_op(get_accum_pkt->data, get_accum_pkt->addr, + mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { + MPIU_Assert(get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM); + mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1052,7 +1166,9 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; + if (!(get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); + } get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -1063,29 +1179,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; - get_accum_resp_pkt->immed_len = 0; - - /* length of target data */ - MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_accum_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_accum_resp_pkt->immed_len > 0) { - void *src = sreq->dev.user_buf; - void *dest = (void*) get_accum_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_accum_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } - - if (len == (size_t)get_accum_resp_pkt->immed_len) { /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); @@ -1094,8 +1189,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)sreq->dev.user_buf + get_accum_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = get_accum_pkt->count * type_size - get_accum_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)sreq->dev.user_buf); + iov[1].MPID_IOV_LEN = get_accum_pkt->count * type_size; iovcnt = 2; } @@ -1118,12 +1213,23 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; MPIDI_CH3_Pkt_fop_t *fop_pkt = &((lock_entry->pkt).fop); MPID_Request *resp_req = NULL; + MPI_Aint type_size; + MPID_IOV iov[MPID_IOV_LIMIT]; + int iovcnt; int mpi_errno = MPI_SUCCESS; /* FIXME: this function is same with PktHandler_FOP(), should do code refactoring on both of them. */ + MPID_Datatype_get_size_macro(fop_pkt->datatype, type_size); + + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED); + } + else { MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); + } + fop_resp_pkt->request_handle = fop_pkt->request_handle; fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -1134,26 +1240,57 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - fop_resp_pkt->immed_len = fop_pkt->immed_len; + + if (!(fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { + resp_req = MPID_Request_create(); + if (resp_req == NULL) { + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomemreq"); + } + MPIU_Object_set_ref(resp_req, 1); + + resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPSendComplete; + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_FOPSendComplete; + + resp_req->dev.target_win_handle = win_ptr->handle; + resp_req->dev.flags = fop_pkt->flags; + + resp_req->dev.user_buf = (void *)MPIU_Malloc(type_size); + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* copy data to resp pkt header */ - void *src = fop_pkt->addr, *dest = fop_resp_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)fop_resp_pkt->immed_len); + void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; + mpi_errno = immed_copy(src, dest, type_size); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + MPIU_Memcpy(resp_req->dev.user_buf, fop_pkt->addr, + type_size); + } /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); int one = 1; - (*uop)(fop_pkt->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { + (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + } + else { + (*uop)(lock_entry->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + } } if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,lock_entry->vc); mpi_errno = MPIDI_CH3_iStartMsg(lock_entry->vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); @@ -1179,9 +1316,24 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPID_Request_release(resp_req); } } + } + else { + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) fop_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*fop_resp_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = type_size; + iovcnt = 2; + + mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, resp_req, iov, iovcnt); + if (mpi_errno != MPI_SUCCESS) { + MPID_Request_release(resp_req); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + goto fn_exit; + } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, fop_pkt->type, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1201,7 +1353,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPI_Aint len; int mpi_errno = MPI_SUCCESS; - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -1220,7 +1372,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len); + MPIU_Memcpy((void *) &cas_resp_pkt->info.data, cas_pkt->addr, len); /* Compare and replace if equal */ if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) { @@ -1257,7 +1409,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, cas_pkt->type, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1297,6 +1449,7 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent /* LOCK+OP packet */ switch(lock_entry->pkt.type) { case (MPIDI_CH3_PKT_PUT): + case (MPIDI_CH3_PKT_PUT_IMMED): mpi_errno = perform_put_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; @@ -1305,18 +1458,21 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; case (MPIDI_CH3_PKT_ACCUMULATE): + case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): mpi_errno = perform_acc_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; case (MPIDI_CH3_PKT_GET_ACCUM): + case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): mpi_errno = perform_get_acc_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; case (MPIDI_CH3_PKT_FOP): + case (MPIDI_CH3_PKT_FOP_IMMED): mpi_errno = perform_fop_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; - case (MPIDI_CH3_PKT_CAS): + case (MPIDI_CH3_PKT_CAS_IMMED): mpi_errno = perform_cas_in_lock_queue(win_ptr, lock_entry); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; diff --git a/src/mpid/ch3/src/ch3u_handle_send_req.c b/src/mpid/ch3/src/ch3u_handle_send_req.c index a1e586b..84ad691 100644 --- a/src/mpid/ch3/src/ch3u_handle_send_req.c +++ b/src/mpid/ch3/src/ch3u_handle_send_req.c @@ -217,7 +217,7 @@ int MPIDI_CH3_ReqHandler_CASSendComplete( MPIDI_VC_t *vc, because inside finish_op_on_target() we may call this request handler on the same request again (in release_lock()). Marking this request as completed will prevent us from processing the same request twice. */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS_IMMED, flags, source_win_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c index d9bdac0..910f297 100644 --- a/src/mpid/ch3/src/ch3u_request.c +++ b/src/mpid/ch3/src/ch3u_request.c @@ -93,7 +93,6 @@ MPID_Request * MPID_Request_create(void) req->dev.OnDataAvail = NULL; req->dev.OnFinal = NULL; req->dev.user_buf = NULL; - req->dev.final_user_buf = NULL; req->dev.drop_data = FALSE; #ifdef MPIDI_CH3_REQUEST_INIT MPIDI_CH3_REQUEST_INIT(req); diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c index 92b4961..3d97174 100644 --- a/src/mpid/ch3/src/ch3u_rma_oplist.c +++ b/src/mpid/ch3/src/ch3u_rma_oplist.c @@ -366,7 +366,9 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe win_ptr->active_req_cnt++; if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending PUT/ACC operation. */ } @@ -401,7 +403,9 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe &(target->dt_op_list_tail), curr_op); } else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT || - curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE) { + curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE || + curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list), &(target->write_op_list_tail), curr_op); } diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 29a65dc..a43a606 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -121,6 +121,9 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_put_t *put_pkt = NULL; + MPI_Aint origin_type_size; + size_t immed_len, len; + int use_immed_pkt = FALSE; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -155,30 +158,48 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); + + /* Judge if we can use IMMED data packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size, + size_t); + if (len <= immed_len) + use_immed_pkt = TRUE; + } + /* Judge if this operation is an piggyback candidate */ if (!new_ptr->is_dt) { - size_t len; - MPI_Aint origin_type_size; - MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t); - if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES, - MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) + if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) new_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ put_pkt = &(new_ptr->pkt.put); - MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + + if (use_immed_pkt) { + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT_IMMED); + } + else { + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + } + put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; put_pkt->count = target_count; put_pkt->datatype = target_datatype; - put_pkt->dataloop_size = 0; + put_pkt->info.dataloop_size = 0; put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; put_pkt->source_win_handle = win_ptr->handle; - put_pkt->immed_len = 0; put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(put_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -288,6 +309,9 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; + MPI_Aint target_type_size; + size_t immed_len, len; + int use_immed_resp_pkt = FALSE; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -322,6 +346,18 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(target_datatype, target_type_size); + MPIU_Assign_trunc(len, target_count * target_type_size, size_t); + + /* Judge if we can use IMMED data response packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (len <= immed_len) + use_immed_resp_pkt = TRUE; + } + /* Judge if this operation is an piggyback candidate. */ if (!new_ptr->is_dt) { new_ptr->piggyback_lock_candidate = 1; @@ -335,10 +371,12 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype win_ptr->disp_units[target_rank] * target_disp; get_pkt->count = target_count; get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; + get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_resp_pkt) + get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -450,6 +488,9 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL; + MPI_Aint origin_type_size; + size_t immed_len, len; + int use_immed_pkt = FALSE; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -483,14 +524,21 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t); + + /* Judge if we can use IMMED data packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size, + size_t); + if (len <= immed_len) + use_immed_pkt = TRUE; + } + /* Judge if this operation is an piggyback candidate. */ if (!new_ptr->is_dt) { - size_t len; - MPI_Aint origin_type_size; - MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t); - if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES, - MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) + if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) new_ptr->piggyback_lock_candidate = 1; } @@ -498,17 +546,27 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp accum_pkt = &(new_ptr->pkt.accum); - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + if (use_immed_pkt) { + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE_IMMED); + } + else { + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + } + accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; accum_pkt->count = target_count; accum_pkt->datatype = target_datatype; - accum_pkt->dataloop_size = 0; + accum_pkt->info.dataloop_size = 0; accum_pkt->op = op; accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; accum_pkt->source_win_handle = win_ptr->handle; - accum_pkt->immed_len = 0; accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(accum_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -634,6 +692,9 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (op == MPI_NO_OP) { /* Convert GAcc to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; + MPI_Aint target_type_size; + size_t len, immed_len; + int use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -659,6 +720,18 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(target_datatype, target_type_size); + MPIU_Assign_trunc(len, target_count * target_type_size, size_t); + + /* Judge if we can use IMMED data response packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (len <= immed_len) + use_immed_resp_pkt = TRUE; + } + /* Judge if this operation is a piggyback candidate */ if (!new_ptr->is_dt) { new_ptr->piggyback_lock_candidate = 1; @@ -672,14 +745,19 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, win_ptr->disp_units[target_rank] * target_disp; get_pkt->count = target_count; get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; + get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_resp_pkt == TRUE) + get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } else { MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; + MPI_Aint origin_type_size, target_type_size; + size_t immed_len, orig_len, tar_len; + int use_immed_pkt = FALSE, use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -715,31 +793,60 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->is_dt = 1; } + MPID_Datatype_get_size_macro(origin_datatype, origin_type_size); + MPIU_Assign_trunc(orig_len, origin_count * origin_type_size, size_t); + + MPID_Datatype_get_size_macro(target_datatype, target_type_size); + MPIU_Assign_trunc(tar_len, target_count * target_type_size, size_t); + + /* Judge if we can use IMMED data packet */ + if (!new_ptr->is_dt) { + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size, + size_t); + if (orig_len <= immed_len) + use_immed_pkt = TRUE; + + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (tar_len <= immed_len) + use_immed_resp_pkt = TRUE; + } + /* Judge if this operation is a piggyback candidate */ if (!new_ptr->is_dt) { - size_t len; - MPI_Aint origin_type_size; - MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size); - MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t); - if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES, - MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) + if (orig_len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE) new_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ get_accum_pkt = &(new_ptr->pkt.get_accum); - MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + + if (use_immed_pkt) { + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED); + } + else { + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + } + get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; get_accum_pkt->count = target_count; get_accum_pkt->datatype = target_datatype; - get_accum_pkt->dataloop_size = 0; + get_accum_pkt->info.dataloop_size = 0; get_accum_pkt->op = op; get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_accum_pkt->source_win_handle = win_ptr->handle; - get_accum_pkt->immed_len = 0; get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(get_accum_pkt->info.data); + mpi_errno = immed_copy(src, dest, orig_len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + if (use_immed_resp_pkt) + get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -953,6 +1060,8 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, else { MPIDI_RMA_Op_t *new_ptr = NULL; MPIDI_CH3_Pkt_cas_t *cas_pkt = NULL; + MPI_Aint type_size; + void *src = NULL, *dest = NULL; /* Append this operation to the RMA ops queue */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr); @@ -976,7 +1085,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, /************** Setting packet struct areas in operation ****************/ cas_pkt = &(new_ptr->pkt.cas); - MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS_IMMED); cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; cas_pkt->datatype = datatype; @@ -984,6 +1093,19 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, cas_pkt->source_win_handle = win_ptr->handle; cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /* REQUIRE: All datatype arguments must be of the same, builtin + * type and counts must be 1. */ + MPID_Datatype_get_size_macro(datatype, type_size); + MPIU_Assert(type_size <= sizeof(MPIDI_CH3_CAS_Immed_u)); + + src = (void *)origin_addr, dest = (void *)(&(cas_pkt->origin_data)); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + src = (void *)compare_addr, dest = (void *)(&(cas_pkt->compare_data)); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); @@ -1090,6 +1212,9 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, if (op == MPI_NO_OP) { /* Convert FOP to a Get */ MPIDI_CH3_Pkt_get_t *get_pkt; + MPI_Aint target_type_size; + size_t immed_len; + int use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -1102,19 +1227,34 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /************** Setting packet struct areas in operation ****************/ + MPID_Datatype_get_size_macro(datatype, target_type_size); + MPIU_Assert(target_type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); + + /* Judege if we can use IMMED data for response packet */ + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/target_type_size)*target_type_size, + size_t); + if (target_type_size <= immed_len) + use_immed_resp_pkt = TRUE; + get_pkt = &(new_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; get_pkt->count = 1; get_pkt->datatype = datatype; - get_pkt->dataloop_size = 0; + get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_resp_pkt == TRUE) + get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } else { MPIDI_CH3_Pkt_fop_t *fop_pkt; + MPI_Aint type_size; + size_t immed_len; + int use_immed_pkt = FALSE, use_immed_resp_pkt = FALSE; /******************** Setting operation struct areas ***********************/ @@ -1129,17 +1269,40 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, /************** Setting packet struct areas in operation ****************/ + MPID_Datatype_get_size_macro(datatype, type_size); + MPIU_Assert(type_size <= sizeof(MPIDI_CH3_FOP_Immed_u)); + + /* Judge if we can use IMMED data packet */ + MPIU_Assign_trunc(immed_len, + (MPIDI_RMA_IMMED_BYTES/type_size)*type_size, + size_t); + if (type_size <= immed_len) { + use_immed_pkt = TRUE; + use_immed_resp_pkt = TRUE; + } + fop_pkt = &(new_ptr->pkt.fop); - MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + if (use_immed_pkt) { + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP_IMMED); + } + else { + MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); + } fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; fop_pkt->datatype = datatype; fop_pkt->op = op; fop_pkt->source_win_handle = win_ptr->handle; fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - fop_pkt->immed_len = 0; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (use_immed_pkt) { + void *src = (void *)origin_addr, *dest = (void *)(fop_pkt->info.data); + mpi_errno = immed_copy(src, dest, type_size); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + if (use_immed_resp_pkt) + fop_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index baa19a6..4163baa 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -214,6 +214,27 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } + MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); + + if (pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { + + /* Immed packet type is used when target datatype is predefined datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)); + + /* copy data from packet header to target buffer */ + MPIU_Memcpy(put_pkt->addr, put_pkt->info.data, put_pkt->count*type_size); + + /* trigger final action */ + mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + put_pkt->flags, put_pkt->source_win_handle); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_PUT); + /* get start location of data and length of data */ data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); @@ -232,28 +253,8 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); req->dev.datatype = put_pkt->datatype; - MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); req->dev.recv_data_sz = type_size * put_pkt->count; - - if (put_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, put_pkt->data, (size_t)put_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + put_pkt->immed_len); - req->dev.recv_data_sz -= put_pkt->immed_len; - } - - if (req->dev.recv_data_sz == 0) { - /* All data received, trigger req handler. */ - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - mpi_errno = MPIDI_CH3_ReqHandler_PutRecvComplete(vc, req, &complete); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } + MPIU_Assert(req->dev.recv_data_sz > 0); mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -284,23 +285,23 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(put_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - put_pkt->dataloop_size); + put_pkt->info.dataloop_size); } /* if we received all of the dtype_info and dataloop, copy it * now and call the handler, otherwise set the iov and let the * channel copy it */ - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + put_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - put_pkt->dataloop_size); + put_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + put_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_PutDerivedDTRecvComplete(vc, req, &complete); @@ -315,7 +316,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) req->dev.dtype_info); req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = put_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = put_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -331,6 +332,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); } + } fn_exit: @@ -351,7 +353,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_CH3_Pkt_get_t *get_pkt = &pkt->get; MPID_Request *req = NULL; MPID_IOV iov[MPID_IOV_LIMIT]; - int complete; + int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; MPID_Win *win_ptr; @@ -391,6 +393,10 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)); + } + if (MPIR_DATATYPE_IS_PREDEFINED(get_pkt->datatype)) { /* basic datatype. send the data. */ MPIDI_CH3_Pkt_t upkt; @@ -403,7 +409,12 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.OnFinal = MPIDI_CH3_ReqHandler_GetSendComplete; req->kind = MPID_REQUEST_SEND; - MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP_IMMED); + } + else { + MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); + } get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || @@ -414,31 +425,16 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_resp_pkt->source_win_handle = get_pkt->source_win_handle; - get_resp_pkt->immed_len = 0; /* length of target data */ MPID_Datatype_get_size_macro(get_pkt->datatype, type_size); - MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t); - /* both origin buffer and target buffer are basic datatype, - fill IMMED data area in response packet header. */ if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { - /* Try to copy target data into packet header. */ - MPIU_Assign_trunc(get_resp_pkt->immed_len, - MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / type_size) * type_size), - int); - - if (get_resp_pkt->immed_len > 0) { - void *src = get_pkt->addr; - void *dest = (void*) get_resp_pkt->data; - /* copy data from origin buffer to immed area in packet header */ - mpi_errno = immed_copy(src, dest, (size_t)get_resp_pkt->immed_len); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - } + MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t); + void *src = (void*)(get_pkt->addr), *dest = (void*)(get_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (len == (size_t)get_resp_pkt->immed_len) { - /* All origin data is in packet header, issue the header. */ iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); iovcnt = 1; @@ -446,8 +442,8 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, else { iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt; iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt); - iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)get_pkt->addr + get_resp_pkt->immed_len); - iov[1].MPID_IOV_LEN = get_pkt->count * type_size - get_resp_pkt->immed_len; + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)get_pkt->addr); + iov[1].MPID_IOV_LEN = get_pkt->count * type_size; iovcnt = 2; } @@ -483,23 +479,23 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(get_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(get_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - get_pkt->dataloop_size); + get_pkt->info.dataloop_size); } /* if we received all of the dtype_info and dataloop, copy it * now and call the handler, otherwise set the iov and let the * channel copy it */ - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - get_pkt->dataloop_size); + get_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete(vc, req, &complete); @@ -512,7 +508,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = get_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = get_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -566,6 +562,31 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } + if (pkt->type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) { + /* Immed packet type is used when target datatype is predefined datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)); + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + mpi_errno = do_accumulate_op(accum_pkt->info.data, accum_pkt->addr, + accum_pkt->count, accum_pkt->datatype, accum_pkt->op); + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + /* trigger final action */ + mpi_errno = finish_op_on_target(win_ptr, vc, pkt->type, + accum_pkt->flags, accum_pkt->source_win_handle); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_ACCUMULATE); + req = MPID_Request_create(); MPIU_Object_set_ref(req, 1); *rreqp = req; @@ -601,28 +622,10 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } req->dev.user_buf = tmp_buf; - req->dev.final_user_buf = req->dev.user_buf; MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size); req->dev.recv_data_sz = type_size * accum_pkt->count; - - if (accum_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, accum_pkt->data, (size_t)accum_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + accum_pkt->immed_len); - req->dev.recv_data_sz -= accum_pkt->immed_len; - } - - if (req->dev.recv_data_sz == 0) { - /* All data received, trigger req handler. */ - *buflen = sizeof(MPIDI_CH3_Pkt_t); - mpi_errno = MPIDI_CH3_ReqHandler_AccumRecvComplete(vc, req, &complete); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } + MPIU_Assert(req->dev.recv_data_sz > 0); mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -653,20 +656,20 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(accum_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - accum_pkt->dataloop_size); + accum_pkt->info.dataloop_size); } - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + accum_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - accum_pkt->dataloop_size); + accum_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + accum_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete(vc, req, &complete); @@ -681,12 +684,13 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = accum_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = accum_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); } } + } if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -741,6 +745,118 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } + MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); + + if (pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { + size_t len; + void *src = NULL, *dest = NULL; + MPID_Request *resp_req = NULL; + MPIDI_CH3_Pkt_t upkt; + MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &upkt.get_accum_resp; + MPID_IOV iov[MPID_IOV_LIMIT]; + int iovcnt; + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + /* Immed packet type is used when target datatype is predefined datatype. */ + MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)); + + resp_req = MPID_Request_create(); + resp_req->dev.target_win_handle = get_accum_pkt->target_win_handle; + resp_req->dev.source_win_handle = get_accum_pkt->source_win_handle; + resp_req->dev.flags = get_accum_pkt->flags; + + MPIDI_Request_set_type(resp_req, MPIDI_REQUEST_TYPE_GET_ACCUM_RESP); + resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumSendComplete; + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumSendComplete; + resp_req->kind = MPID_REQUEST_SEND; + if (!(get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)) { + tmp_buf = MPIU_Malloc(get_accum_pkt->count * type_size); + if (!tmp_buf) { + MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d", + get_accum_pkt->count * type_size); + } + resp_req->dev.user_buf = tmp_buf; + } + + /* here we increment the Active Target counter to guarantee the GET-like + operation are completed when counter reaches zero. */ + win_ptr->at_completion_counter++; + + /* Calculate the length of reponse data, ensure that it fits into immed packet. */ + MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); + + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); + } + else { + MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); + } + + get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; + get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; + get_accum_resp_pkt->source_win_handle = get_accum_pkt->source_win_handle; + get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) + get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; + if ((get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || + (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) + get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { + /* copy data from target buffer to response packet header */ + src = (void *) (get_accum_pkt->addr), dest = (void *) (get_accum_resp_pkt->info.data); + mpi_errno = immed_copy(src, dest, len); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + MPIU_Memcpy(resp_req->dev.user_buf, get_accum_pkt->addr, + get_accum_pkt->count * type_size); + } + + /* perform accumulate operation. */ + mpi_errno = do_accumulate_op(get_accum_pkt->info.data, get_accum_pkt->addr, + get_accum_pkt->count, get_accum_pkt->datatype, + get_accum_pkt->op); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); + iovcnt = 1; + } + else { + iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_accum_resp_pkt; + iov[0].MPID_IOV_LEN = sizeof(*get_accum_resp_pkt); + iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *)resp_req->dev.user_buf); + iov[1].MPID_IOV_LEN = get_accum_pkt->count * type_size; + iovcnt = 2; + } + + MPIU_THREAD_CS_ENTER(CH3COMM, vc); + mpi_errno = MPIDI_CH3_iSendv(vc, resp_req, iov, iovcnt); + MPIU_THREAD_CS_EXIT(CH3COMM, vc); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPIU_Object_set_ref(resp_req, 0); + MPIDI_CH3_Request_destroy(resp_req); + MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); + } + /* --END ERROR HANDLING-- */ + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_GET_ACCUM); + req = MPID_Request_create(); MPIU_Object_set_ref(req, 1); *rreqp = req; @@ -776,30 +892,9 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } req->dev.user_buf = tmp_buf; - req->dev.final_user_buf = req->dev.user_buf; - MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); req->dev.recv_data_sz = type_size * get_accum_pkt->count; - - if (get_accum_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, get_accum_pkt->data, (size_t)get_accum_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + get_accum_pkt->immed_len); - req->dev.recv_data_sz -= get_accum_pkt->immed_len; - } - - if (req->dev.recv_data_sz == 0) { - /* All data received. */ - - *buflen = sizeof(MPIDI_CH3_Pkt_t); - - mpi_errno = MPIDI_CH3_ReqHandler_GaccumRecvComplete(vc, req, &complete); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - if (complete) { - *rreqp = NULL; - goto fn_exit; - } - } + MPIU_Assert(req->dev.recv_data_sz > 0); mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", @@ -830,20 +925,20 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, "MPIDI_RMA_dtype_info"); } - req->dev.dataloop = MPIU_Malloc(get_accum_pkt->dataloop_size); + req->dev.dataloop = MPIU_Malloc(get_accum_pkt->info.dataloop_size); if (!req->dev.dataloop) { MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", - get_accum_pkt->dataloop_size); + get_accum_pkt->info.dataloop_size); } - if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->dataloop_size) { + if (data_len >= sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->info.dataloop_size) { /* copy all of dtype_info and dataloop */ MPIU_Memcpy(req->dev.dtype_info, data_buf, sizeof(MPIDI_RMA_dtype_info)); MPIU_Memcpy(req->dev.dataloop, data_buf + sizeof(MPIDI_RMA_dtype_info), - get_accum_pkt->dataloop_size); + get_accum_pkt->info.dataloop_size); *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->dataloop_size; + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_RMA_dtype_info) + get_accum_pkt->info.dataloop_size; /* All dtype data has been received, call req handler */ mpi_errno = MPIDI_CH3_ReqHandler_GaccumDerivedDTRecvComplete(vc, req, &complete); @@ -858,7 +953,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dtype_info; req->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_RMA_dtype_info); req->dev.iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) req->dev.dataloop; - req->dev.iov[1].MPID_IOV_LEN = get_accum_pkt->dataloop_size; + req->dev.iov[1].MPID_IOV_LEN = get_accum_pkt->info.dataloop_size; req->dev.iov_count = 2; *buflen = sizeof(MPIDI_CH3_Pkt_t); } @@ -869,6 +964,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); } + } fn_exit: MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_accum); @@ -923,7 +1019,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, *buflen = sizeof(MPIDI_CH3_Pkt_t); *rreqp = NULL; - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); + MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP_IMMED); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -942,7 +1038,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - MPIU_Memcpy((void *) &cas_resp_pkt->data, cas_pkt->addr, len); + MPIU_Memcpy((void *) &cas_resp_pkt->info.data, cas_pkt->addr, len); /* Compare and replace if equal */ if (MPIR_Compare_equal(&cas_pkt->compare_data, cas_pkt->addr, cas_pkt->datatype)) { @@ -978,7 +1074,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Request_release(req); } - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, vc, cas_pkt->type, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1034,7 +1130,7 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPID_Request_get_ptr(cas_resp_pkt->request_handle, req); MPID_Datatype_get_size_macro(req->dev.datatype, len); - MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->data, len); + MPIU_Memcpy(req->dev.user_buf, (void *) &cas_resp_pkt->info.data, len); MPIDI_CH3U_Request_complete(req); *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -1064,6 +1160,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Request *rreq = NULL; int acquire_lock_fail = 0; MPID_Win *win_ptr = NULL; + MPI_Aint type_size; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP); @@ -1088,7 +1185,13 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, (*buflen) = sizeof(MPIDI_CH3_Pkt_t); (*rreqp) = NULL; - MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); + MPID_Datatype_get_size_macro(fop_pkt->datatype, type_size); + + if (pkt->type == MPIDI_CH3_PKT_FOP_IMMED) { + + MPIU_Assert(fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP); + + MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED); fop_resp_pkt->request_handle = fop_pkt->request_handle; fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; @@ -1099,21 +1202,20 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; - fop_resp_pkt->immed_len = fop_pkt->immed_len; if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); /* copy data to resp pkt header */ - void *src = fop_pkt->addr, *dest = fop_resp_pkt->data; - mpi_errno = immed_copy(src, dest, (size_t)fop_resp_pkt->immed_len); + void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data; + mpi_errno = immed_copy(src, dest, type_size); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); /* Apply the op */ if (fop_pkt->op != MPI_NO_OP) { MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); int one = 1; - (*uop)(fop_pkt->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); + (*uop)(fop_pkt->info.data, fop_pkt->addr, &one, &(fop_pkt->datatype)); } if (win_ptr->shm_allocated == TRUE) @@ -1145,9 +1247,68 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } } - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, vc, fop_pkt->type, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_FOP); + + MPIU_Assert(!(fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP)); + + MPID_Request *req = NULL; + char *data_buf = NULL; + MPIDI_msg_sz_t data_len; + MPI_Aint extent; + int complete = 0; + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); + *rreqp = req; + + req->dev.op = fop_pkt->op; + req->dev.real_user_buf = fop_pkt->addr; + req->dev.target_win_handle = fop_pkt->target_win_handle; + req->dev.source_win_handle = fop_pkt->source_win_handle; + req->dev.flags = fop_pkt->flags; + req->dev.resp_request_handle = fop_pkt->request_handle; + req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_FOPRecvComplete; + req->dev.OnFinal = MPIDI_CH3_ReqHandler_FOPRecvComplete; + req->dev.datatype = fop_pkt->datatype; + req->dev.user_count = 1; + + /* get start location of data and length of data */ + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + MPID_Datatype_get_extent_macro(fop_pkt->datatype, extent); + + req->dev.user_buf = MPIU_Malloc(extent); + if (!req->dev.user_buf) { + MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %d", + extent); + } + + req->dev.recv_data_sz = type_size; + MPIU_Assert(req->dev.recv_data_sz > 0); + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", + "**ch3|postrecv %s", "MPIDI_CH3_PKT_ACCUMULATE"); + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + + if (complete) { + mpi_errno = MPIDI_CH3_ReqHandler_FOPRecvComplete(vc, req, &complete); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + if (complete) { + *rreqp = NULL; + goto fn_exit; + } + } + } fn_exit: MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop); @@ -1172,6 +1333,10 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &pkt->fop_resp; MPID_Request *req = NULL; MPID_Win *win_ptr = NULL; + MPI_Aint type_size; + MPIDI_msg_sz_t data_len; + char *data_buf = NULL; + int complete = 0; int target_rank = fop_resp_pkt->target_rank; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP); @@ -1183,9 +1348,7 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPID_Win_get_ptr(fop_resp_pkt->source_win_handle, win_ptr); - /* Copy data to result buffer on orgin */ MPID_Request_get_ptr(fop_resp_pkt->request_handle, req); - MPIU_Memcpy(req->dev.user_buf, fop_resp_pkt->data, (size_t)fop_resp_pkt->immed_len); /* decrement ack_counter */ if (fop_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { @@ -1202,9 +1365,37 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), if (mpi_errno) MPIU_ERR_POP(mpi_errno); } + data_len = *buflen - sizeof(MPIDI_CH3_Pkt_t); + data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); + + MPID_Datatype_get_size_macro(req->dev.datatype, type_size); + req->dev.recv_data_sz = type_size; + req->dev.user_count = 1; + + *rreqp = req; + + if (fop_resp_pkt->type == MPIDI_CH3_PKT_FOP_RESP_IMMED) { + MPIU_Memcpy(req->dev.user_buf, fop_resp_pkt->info.data, req->dev.recv_data_sz); + + /* return the number of bytes processed in this function */ + *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; + } + else { + MPIU_Assert(fop_resp_pkt->type == MPIDI_CH3_PKT_FOP_RESP); + + mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); + MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", + "MPIDI_CH3_PKT_FOP_RESP"); + + /* return the number of bytes processed in this function */ + *buflen = data_len + sizeof(MPIDI_CH3_Pkt_t); + } + + if (complete) { MPIDI_CH3U_Request_complete(req); - *buflen = sizeof(MPIDI_CH3_Pkt_t); *rreqp = NULL; + } fn_exit: MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop_resp); @@ -1226,7 +1417,7 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &pkt->get_accum_resp; MPID_Request *req; - int complete; + int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; int mpi_errno = MPI_SUCCESS; @@ -1266,20 +1457,17 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; - if (get_accum_resp_pkt->immed_len > 0) { - /* first copy IMMED data from pkt header to origin buffer */ - MPIU_Memcpy(req->dev.user_buf, get_accum_resp_pkt->data, (size_t)get_accum_resp_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + get_accum_resp_pkt->immed_len); - req->dev.recv_data_sz -= get_accum_resp_pkt->immed_len; - if (req->dev.recv_data_sz == 0) - complete = 1; + *rreqp = req; + if (get_accum_resp_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED) { + MPIU_Memcpy(req->dev.user_buf, get_accum_resp_pkt->info.data, req->dev.recv_data_sz); /* return the number of bytes processed in this function */ *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; } + else { + MPIU_Assert(pkt->type == MPIDI_CH3_PKT_GET_ACCUM_RESP); - if(req->dev.recv_data_sz > 0) { - *rreqp = req; mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_ACCUM_RESP"); @@ -1371,7 +1559,7 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), { MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &pkt->get_resp; MPID_Request *req; - int complete; + int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; int mpi_errno = MPI_SUCCESS; @@ -1411,20 +1599,18 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)), MPID_Datatype_get_size_macro(req->dev.datatype, type_size); req->dev.recv_data_sz = type_size * req->dev.user_count; - if (get_resp_pkt->immed_len > 0) { - /* first copy IMMED data from pkt header to origin buffer */ - MPIU_Memcpy(req->dev.user_buf, get_resp_pkt->data, (size_t)get_resp_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + get_resp_pkt->immed_len); - req->dev.recv_data_sz -= get_resp_pkt->immed_len; - if (req->dev.recv_data_sz == 0) - complete = 1; + *rreqp = req; + + if (get_resp_pkt->type == MPIDI_CH3_PKT_GET_RESP_IMMED) { + MPIU_Memcpy(req->dev.user_buf, get_resp_pkt->info.data, req->dev.recv_data_sz); /* return the number of bytes processed in this function */ *buflen = sizeof(MPIDI_CH3_Pkt_t); + complete = 1; } + else { + MPIU_Assert(get_resp_pkt->type == MPIDI_CH3_PKT_GET_RESP); - if (req->dev.recv_data_sz > 0) { - *rreqp = req; mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len, &complete); MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", "**ch3|postrecv %s", "MPIDI_CH3_PKT_GET_RESP"); @@ -1705,7 +1891,7 @@ int MPIDI_CH3_PktPrint_Put(FILE * fp, MPIDI_CH3_Pkt_t * pkt) MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->put.addr)); MPIU_DBG_PRINTF((" count ........ %d\n", pkt->put.count)); MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->put.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.dataloop_size)); + MPIU_DBG_PRINTF((" dataloop_size. 0x%08X\n", pkt->put.info.dataloop_size)); MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->put.target_win_handle)); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->put.source_win_handle)); /*MPIU_DBG_PRINTF((" win_ptr ...... 0x%08X\n", pkt->put.win_ptr)); */ @@ -1718,7 +1904,7 @@ int MPIDI_CH3_PktPrint_Get(FILE * fp, MPIDI_CH3_Pkt_t * pkt) MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->get.addr)); MPIU_DBG_PRINTF((" count ........ %d\n", pkt->get.count)); MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->get.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.dataloop_size)); + MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->get.info.dataloop_size)); MPIU_DBG_PRINTF((" request ...... 0x%08X\n", pkt->get.request_handle)); MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->get.target_win_handle)); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->get.source_win_handle)); @@ -1743,7 +1929,7 @@ int MPIDI_CH3_PktPrint_Accumulate(FILE * fp, MPIDI_CH3_Pkt_t * pkt) MPIU_DBG_PRINTF((" addr ......... %p\n", pkt->accum.addr)); MPIU_DBG_PRINTF((" count ........ %d\n", pkt->accum.count)); MPIU_DBG_PRINTF((" datatype ..... 0x%08X\n", pkt->accum.datatype)); - MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.dataloop_size)); + MPIU_DBG_PRINTF((" dataloop_size. %d\n", pkt->accum.info.dataloop_size)); MPIU_DBG_PRINTF((" op ........... 0x%08X\n", pkt->accum.op)); MPIU_DBG_PRINTF((" target ....... 0x%08X\n", pkt->accum.target_win_handle)); MPIU_DBG_PRINTF((" source ....... 0x%08X\n", pkt->accum.source_win_handle)); http://git.mpich.org/mpich.git/commitdiff/3a017faae893f03d2ad2fb4d24f037d96… commit 3a017faae893f03d2ad2fb4d24f037d964782e23 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Fri Jan 23 10:28:46 2015 -0600 Code-refactoring for RMA operations routines. This patch just does code refactoring for RMA operation rountines to make the code structure clearer. This patch does not change any functionality. After code refactoring, in each operation routine, for non-SHM operations we do the work in the following order: (1) allocate a new op struct; (2) fill areas in op struct, except for packet struct in op struct; (3) initialize packet struct in op struct, fill areas in packet struct; (4) enqueue op to data structure on window. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index 45df32b..29a65dc 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -128,17 +128,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - put_pkt = &(new_ptr->pkt.put); - MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); - put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - put_pkt->count = target_count; - put_pkt->datatype = target_datatype; - put_pkt->dataloop_size = 0; - put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - put_pkt->source_win_handle = win_ptr->handle; - put_pkt->immed_len = 0; - put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->origin_addr = (void *) origin_addr; @@ -152,14 +142,6 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->ureq = ureq; } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -184,8 +166,26 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype new_ptr->piggyback_lock_candidate = 1; } + /************** Setting packet struct areas in operation ****************/ + + put_pkt = &(new_ptr->pkt.put); + MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT); + put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + put_pkt->count = target_count; + put_pkt->datatype = target_datatype; + put_pkt->dataloop_size = 0; + put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + put_pkt->source_win_handle = win_ptr->handle; + put_pkt->immed_len = 0; + put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -295,16 +295,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - get_pkt = &(new_ptr->pkt.get); - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - get_pkt->count = target_count; - get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; - get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; - get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->origin_addr = origin_addr; @@ -318,14 +309,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->ureq = ureq; } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -344,8 +327,25 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype new_ptr->piggyback_lock_candidate = 1; } + /************** Setting packet struct areas in operation ****************/ + + get_pkt = &(new_ptr->pkt.get); + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_pkt->count = target_count; + get_pkt->datatype = target_datatype; + get_pkt->dataloop_size = 0; + get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_pkt->source_win_handle = win_ptr->handle; + get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -457,19 +457,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - accum_pkt = &(new_ptr->pkt.accum); - - MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); - accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - accum_pkt->count = target_count; - accum_pkt->datatype = target_datatype; - accum_pkt->dataloop_size = 0; - accum_pkt->op = op; - accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - accum_pkt->source_win_handle = win_ptr->handle; - accum_pkt->immed_len = 0; - accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = origin_count; @@ -482,14 +470,6 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp new_ptr->ureq = ureq; } - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); - - mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - - MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { @@ -514,8 +494,28 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp new_ptr->piggyback_lock_candidate = 1; } + /************** Setting packet struct areas in operation ****************/ + + accum_pkt = &(new_ptr->pkt.accum); + + MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE); + accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + accum_pkt->count = target_count; + accum_pkt->datatype = target_datatype; + accum_pkt->dataloop_size = 0; + accum_pkt->op = op; + accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + accum_pkt->source_win_handle = win_ptr->handle; + accum_pkt->immed_len = 0; + accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); + mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); + if (mpi_errno) + MPIU_ERR_POP(mpi_errno); + mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -633,21 +633,20 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (op == MPI_NO_OP) { /* Convert GAcc to a Get */ - MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get); - MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); - get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - get_pkt->count = target_count; - get_pkt->datatype = target_datatype; - get_pkt->dataloop_size = 0; - get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_pkt->source_win_handle = win_ptr->handle; - get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIDI_CH3_Pkt_get_t *get_pkt; + + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = result_addr; new_ptr->origin_count = result_count; new_ptr->origin_datatype = result_datatype; new_ptr->target_rank = target_rank; + new_ptr->ureq = NULL; /* reset user request */ + + /* Remember user request */ + if (ureq) { + new_ptr->ureq = ureq; + } if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) { MPID_Datatype_get_ptr(result_datatype, dtp); @@ -664,21 +663,25 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, if (!new_ptr->is_dt) { new_ptr->piggyback_lock_candidate = 1; } + + /************** Setting packet struct areas in operation ****************/ + + get_pkt = &(new_ptr->pkt.get); + MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); + get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_pkt->count = target_count; + get_pkt->datatype = target_datatype; + get_pkt->dataloop_size = 0; + get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_pkt->source_win_handle = win_ptr->handle; + get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; } else { - MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt = &(new_ptr->pkt.get_accum); - MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); - get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - get_accum_pkt->count = target_count; - get_accum_pkt->datatype = target_datatype; - get_accum_pkt->dataloop_size = 0; - get_accum_pkt->op = op; - get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - get_accum_pkt->source_win_handle = win_ptr->handle; - get_accum_pkt->immed_len = 0; - get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt; + + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = origin_count; @@ -687,6 +690,12 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, new_ptr->result_count = result_count; new_ptr->result_datatype = result_datatype; new_ptr->target_rank = target_rank; + new_ptr->ureq = NULL; /* reset user request */ + + /* Remember user request */ + if (ureq) { + new_ptr->ureq = ureq; + } /* if source or target datatypes are derived, increment their * reference counts */ @@ -716,13 +725,21 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)) new_ptr->piggyback_lock_candidate = 1; } - } - new_ptr->ureq = NULL; /* reset user request */ + /************** Setting packet struct areas in operation ****************/ - /* Remember user request */ - if (ureq) { - new_ptr->ureq = ureq; + get_accum_pkt = &(new_ptr->pkt.get_accum); + MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM); + get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + get_accum_pkt->count = target_count; + get_accum_pkt->datatype = target_datatype; + get_accum_pkt->dataloop_size = 0; + get_accum_pkt->op = op; + get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + get_accum_pkt->source_win_handle = win_ptr->handle; + get_accum_pkt->immed_len = 0; + get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; } MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); @@ -943,14 +960,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); - cas_pkt = &(new_ptr->pkt.cas); - MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); - cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + - win_ptr->disp_units[target_rank] * target_disp; - cas_pkt->datatype = datatype; - cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; - cas_pkt->source_win_handle = win_ptr->handle; - cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + /******************** Setting operation struct areas ***********************/ new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; @@ -963,6 +973,17 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, new_ptr->piggyback_lock_candidate = 1; /* CAS is always able to piggyback LOCK */ new_ptr->ureq = NULL; /* reset user request */ + /************** Setting packet struct areas in operation ****************/ + + cas_pkt = &(new_ptr->pkt.cas); + MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS); + cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + + win_ptr->disp_units[target_rank] * target_disp; + cas_pkt->datatype = datatype; + cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; + cas_pkt->source_win_handle = win_ptr->handle; + cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); @@ -1068,7 +1089,20 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, if (op == MPI_NO_OP) { /* Convert FOP to a Get */ - MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get); + MPIDI_CH3_Pkt_get_t *get_pkt; + + /******************** Setting operation struct areas ***********************/ + + new_ptr->origin_addr = result_addr; + new_ptr->origin_count = 1; + new_ptr->origin_datatype = datatype; + new_ptr->target_rank = target_rank; + new_ptr->piggyback_lock_candidate = 1; + new_ptr->ureq = NULL; /* reset user request */ + + /************** Setting packet struct areas in operation ****************/ + + get_pkt = &(new_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + win_ptr->disp_units[target_rank] * target_disp; @@ -1078,15 +1112,24 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + } + else { + MPIDI_CH3_Pkt_fop_t *fop_pkt; - new_ptr->origin_addr = result_addr; + /******************** Setting operation struct areas ***********************/ + + new_ptr->origin_addr = (void *) origin_addr; new_ptr->origin_count = 1; new_ptr->origin_datatype = datatype; + new_ptr->result_addr = result_addr; + new_ptr->result_datatype = datatype; new_ptr->target_rank = target_rank; new_ptr->piggyback_lock_candidate = 1; - } - else { - MPIDI_CH3_Pkt_fop_t *fop_pkt = &(new_ptr->pkt.fop); + new_ptr->ureq = NULL; /* reset user request */ + + /************** Setting packet struct areas in operation ****************/ + + fop_pkt = &(new_ptr->pkt.fop); MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP); fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] + @@ -1097,18 +1140,8 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; fop_pkt->immed_len = 0; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - - new_ptr->origin_addr = (void *) origin_addr; - new_ptr->origin_count = 1; - new_ptr->origin_datatype = datatype; - new_ptr->result_addr = result_addr; - new_ptr->result_datatype = datatype; - new_ptr->target_rank = target_rank; - new_ptr->piggyback_lock_candidate = 1; } - new_ptr->ureq = NULL; /* reset user request */ - MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr); http://git.mpich.org/mpich.git/commitdiff/81e2b2742b9a690e68d40d93e42a6d368… commit 81e2b2742b9a690e68d40d93e42a6d3684206fa3 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Mon Feb 9 17:04:22 2015 -0800 Remove lock_type and origin_rank areas from RMA packet. Originally we added lock_type and origin_rank areas in RMA packet, in order to piggyback passive lock request with RMA operations. However, those areas potentially enlarged the packet union size, and actually they are not necessary and can be completetly avoided. "Lock_type" is used to remember what types of lock (shared or exclusive) the origin wants to acquire on the target. To remove it from RMA packet, we use flags (already exists in RMA packet) to remember such information. "Origin_rank" is used to remember which origin has sent lock request to the target, so that when the lock is granted to this origin later, the target can send ack to that origin. Actually the target does not need to store origin_rank but can only store origin_vc, which is known from progress engine on target side. Therefore, we can completely remove origin_rank from RMA packet. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpid_rma_issue.h b/src/mpid/ch3/include/mpid_rma_issue.h index c64a226..a443521 100644 --- a/src/mpid/ch3/include/mpid_rma_issue.h +++ b/src/mpid/ch3/include/mpid_rma_issue.h @@ -339,8 +339,6 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr, rma_op->request = NULL; put_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - put_pkt->lock_type = target_ptr->lock_type; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); @@ -413,9 +411,6 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, accum_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - accum_pkt->lock_type = target_ptr->lock_type; - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); @@ -516,9 +511,6 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr, get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - get_accum_pkt->lock_type = target_ptr->lock_type; - MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); @@ -650,9 +642,6 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr, get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - get_pkt->lock_type = target_ptr->lock_type; - comm_ptr = win_ptr->comm_ptr; MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc); @@ -750,8 +739,6 @@ static int issue_cas_op(MPIDI_RMA_Op_t * rma_op, cas_pkt->request_handle = rma_op->request->handle; cas_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - cas_pkt->lock_type = target_ptr->lock_type; MPIU_Memcpy((void *) &cas_pkt->origin_data, rma_op->origin_addr, len); MPIU_Memcpy((void *) &cas_pkt->compare_data, rma_op->compare_addr, len); @@ -821,8 +808,6 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op, fop_pkt->request_handle = resp_req->handle; fop_pkt->flags |= flags; - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) - fop_pkt->lock_type = target_ptr->lock_type; MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size); MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t); diff --git a/src/mpid/ch3/include/mpid_rma_lockqueue.h b/src/mpid/ch3/include/mpid_rma_lockqueue.h index 0e1919b..1c98ced 100644 --- a/src/mpid/ch3/include/mpid_rma_lockqueue.h +++ b/src/mpid/ch3/include/mpid_rma_lockqueue.h @@ -32,6 +32,7 @@ static inline MPIDI_RMA_Lock_entry_t *MPIDI_CH3I_Win_lock_entry_alloc(MPID_Win * if (new_ptr != NULL) { new_ptr->next = NULL; new_ptr->pkt = (*pkt); + new_ptr->vc = NULL; new_ptr->data = NULL; new_ptr->data_size = 0; new_ptr->all_data_recved = 0; diff --git a/src/mpid/ch3/include/mpid_rma_types.h b/src/mpid/ch3/include/mpid_rma_types.h index 9089b1a..a1060d7 100644 --- a/src/mpid/ch3/include/mpid_rma_types.h +++ b/src/mpid/ch3/include/mpid_rma_types.h @@ -131,6 +131,7 @@ extern MPIDI_RMA_Win_list_t *MPIDI_RMA_Win_list, *MPIDI_RMA_Win_list_tail; typedef struct MPIDI_RMA_Lock_entry { struct MPIDI_RMA_Lock_entry *next; MPIDI_CH3_Pkt_t pkt; /* all information for this request packet */ + MPIDI_VC_t *vc; void *data; /* for queued PUTs / ACCs / GACCs, data is copied here */ int data_size; int all_data_recved; /* indicate if all data has been received */ diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 852160b..f6e5327 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -109,21 +109,20 @@ typedef enum { /* These flags can be "OR'ed" together */ typedef enum { MPIDI_CH3_PKT_FLAG_NONE = 0, - MPIDI_CH3_PKT_FLAG_RMA_LOCK = 1, - MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 2, - MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 4, - MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 8, - MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 16, - MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 32, - MPIDI_CH3_PKT_FLAG_RMA_SHARED = 64, - MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE = 128, - MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 256, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 512, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 1024, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 2048, - MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 4096, - MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 8192, - MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 16384 + MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2, + MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4, + MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8, + MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16, + MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32, + MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64, + MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 128, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024, + MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048, + MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096, + MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192 } MPIDI_CH3_Pkt_flags_t; typedef struct MPIDI_CH3_Pkt_send { @@ -330,72 +329,6 @@ MPIDI_CH3_PKT_DEFS } \ } -#define MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(pkt_, lock_type_, err_) \ - { \ - /* This macro returns lock type in RMA operation \ - packets (PUT, GET, ACC, GACC, FOP, CAS) and RMA control \ - packets (LOCK). */ \ - err_ = MPI_SUCCESS; \ - switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - lock_type_ = (pkt_).put.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_GET): \ - lock_type_ = (pkt_).get.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - lock_type_ = (pkt_).accum.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - lock_type_ = (pkt_).get_accum.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_CAS): \ - lock_type_ = (pkt_).cas.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - lock_type_ = (pkt_).fop.lock_type; \ - break; \ - case (MPIDI_CH3_PKT_LOCK): \ - lock_type_ = (pkt_).lock.lock_type; \ - break; \ - default: \ - MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ - } \ - } - -#define MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK(pkt_, origin_rank_, err_) \ - { \ - /* This macro returns origin rank (used in acquiring lock) in \ - RMA operation packets (PUT, GET, ACC, GACC, FOP, CAS) and \ - RMA control packets (LOCK). */ \ - err_ = MPI_SUCCESS; \ - switch((pkt_).type) { \ - case (MPIDI_CH3_PKT_PUT): \ - origin_rank_ = (pkt_).put.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_GET): \ - origin_rank_ = (pkt_).get.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_ACCUMULATE): \ - origin_rank_ = (pkt_).accum.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_GET_ACCUM): \ - origin_rank_ = (pkt_).get_accum.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_CAS): \ - origin_rank_ = (pkt_).cas.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_FOP): \ - origin_rank_ = (pkt_).fop.origin_rank; \ - break; \ - case (MPIDI_CH3_PKT_LOCK): \ - origin_rank_ = (pkt_).lock.origin_rank; \ - break; \ - default: \ - MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ - } \ - } - #define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_) \ { \ /* This macro returns flags in RMA operation packets (PUT, GET, \ @@ -435,6 +368,9 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_CAS_RESP): \ flags_ = (pkt_).cas_resp.flags; \ break; \ + case (MPIDI_CH3_PKT_LOCK): \ + flags_ = (pkt_).lock.flags; \ + break; \ case (MPIDI_CH3_PKT_UNLOCK): \ flags_ = (pkt_).unlock.flags; \ break; \ @@ -488,6 +424,9 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_CAS_RESP): \ (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ + case (MPIDI_CH3_PKT_LOCK): \ + (pkt_).lock.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ case (MPIDI_CH3_PKT_UNLOCK): \ (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ @@ -646,8 +585,6 @@ typedef struct MPIDI_CH3_Pkt_put { * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_put_t; typedef struct MPIDI_CH3_Pkt_get { @@ -665,8 +602,6 @@ typedef struct MPIDI_CH3_Pkt_get { MPI_Win source_win_handle; /* Used in the last RMA operation in an * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_t; typedef struct MPIDI_CH3_Pkt_get_resp { @@ -698,8 +633,6 @@ typedef struct MPIDI_CH3_Pkt_accum { * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum { @@ -720,8 +653,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum { * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum_resp { @@ -749,8 +680,6 @@ typedef struct MPIDI_CH3_Pkt_cas { * in passive target rma. Otherwise set to NULL*/ MPIDI_CH3_CAS_Immed_u origin_data; MPIDI_CH3_CAS_Immed_u compare_data; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_cas_t; typedef struct MPIDI_CH3_Pkt_cas_resp { @@ -777,8 +706,6 @@ typedef struct MPIDI_CH3_Pkt_fop { * in passive target rma. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; int immed_len; - int lock_type; /* used when piggybacking LOCK message. */ - int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_fop_t; typedef struct MPIDI_CH3_Pkt_fop_resp { @@ -794,10 +721,9 @@ typedef struct MPIDI_CH3_Pkt_fop_resp { typedef struct MPIDI_CH3_Pkt_lock { MPIDI_CH3_Pkt_type_t type; - int lock_type; + MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; - int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { diff --git a/src/mpid/ch3/include/mpidrma.h b/src/mpid/ch3/include/mpidrma.h index 6d9594d..d7a720d 100644 --- a/src/mpid/ch3/include/mpidrma.h +++ b/src/mpid/ch3/include/mpidrma.h @@ -32,8 +32,13 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr) MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = win_ptr->all_win_handles[dest]; lock_pkt->source_win_handle = win_ptr->handle; - lock_pkt->lock_type = lock_type; - lock_pkt->origin_rank = win_ptr->comm_ptr->rank; + lock_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (lock_type == MPI_LOCK_SHARED) + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; + else { + MPIU_Assert(lock_type == MPI_LOCK_EXCLUSIVE); + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE; + } MPIU_THREAD_CS_ENTER(CH3COMM, vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, lock_pkt, sizeof(*lock_pkt), &req); @@ -321,6 +326,7 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, new_ptr = MPIDI_CH3I_Win_lock_entry_alloc(win_ptr, pkt); if (new_ptr != NULL) { MPL_LL_APPEND(win_ptr->lock_queue, win_ptr->lock_queue_tail, new_ptr); + new_ptr->vc = vc; } else { lock_discarded = 1; @@ -386,18 +392,16 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t new_pkt; MPIDI_CH3_Pkt_lock_t *lock_pkt = &new_pkt.lock; MPI_Win target_win_handle; - int lock_type, origin_rank; + MPIDI_CH3_Pkt_flags_t flags; MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE((*pkt), target_win_handle, mpi_errno); MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE((*pkt), source_win_handle, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK((*pkt), origin_rank, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE((*pkt), lock_type, mpi_errno); + MPIDI_CH3_PKT_RMA_GET_FLAGS((*pkt), flags, mpi_errno); MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); lock_pkt->target_win_handle = target_win_handle; lock_pkt->source_win_handle = source_win_handle; - lock_pkt->lock_type = lock_type; - lock_pkt->origin_rank = origin_rank; + lock_pkt->flags = flags; /* replace original pkt with lock pkt */ new_ptr->pkt = new_pkt; @@ -571,7 +575,8 @@ static inline int adjust_op_piggybacked_with_lock (MPID_Win *win_ptr, op = target->pending_op_list; if (op != NULL) MPIDI_CH3_PKT_RMA_GET_FLAGS(op->pkt, op_flags, mpi_errno); - if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { + if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED || flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED) { if (!op->request) { @@ -664,10 +669,16 @@ static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type) MPIDI_CH3_Pkt_t pkt; MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt.lock; MPIDI_RMA_Lock_entry_t *new_ptr = NULL; + MPIDI_VC_t *my_vc; MPIDI_Pkt_init(lock_pkt, MPIDI_CH3_PKT_LOCK); - lock_pkt->lock_type = lock_type; - lock_pkt->origin_rank = win_ptr->comm_ptr->rank; + lock_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; + if (lock_type == MPI_LOCK_SHARED) + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; + else { + MPIU_Assert(lock_type == MPI_LOCK_EXCLUSIVE); + lock_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE; + } new_ptr = MPIDI_CH3I_Win_lock_entry_alloc(win_ptr, &pkt); if (new_ptr == NULL) { @@ -677,6 +688,8 @@ static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type) goto fn_exit; } MPL_LL_APPEND(win_ptr->lock_queue, win_ptr->lock_queue_tail, new_ptr); + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &my_vc); + new_ptr->vc = my_vc; new_ptr->all_data_recved = 1; } @@ -834,9 +847,16 @@ static inline int check_piggyback_lock(MPID_Win *win_ptr, MPIDI_VC_t *vc, (*reqp) = NULL; MPIDI_CH3_PKT_RMA_GET_FLAGS((*pkt), flags, mpi_errno); - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE((*pkt), lock_type, mpi_errno); + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { + + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) + lock_type = MPI_LOCK_SHARED; + else { + MPIU_Assert(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + lock_type = MPI_LOCK_EXCLUSIVE; + } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 0) { /* cannot acquire the lock, queue up this operation. */ mpi_errno = enqueue_lock_origin(win_ptr, vc, pkt, buflen, reqp); @@ -859,7 +879,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, if (type == MPIDI_CH3_PKT_PUT || type == MPIDI_CH3_PKT_ACCUMULATE) { /* This is PUT or ACC */ - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { MPIDI_CH3_Pkt_flags_t pkt_flags = MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -871,7 +892,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Progress_signal_completion(); } if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) { - if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)) { + if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE)) { /* If op is piggybacked with both LOCK and FLUSH, we only send LOCK ACK back, do not send FLUSH ACK. */ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, @@ -888,7 +910,8 @@ static inline int finish_op_on_target(MPID_Win *win_ptr, MPIDI_VC_t *vc, MPIDI_CH3_Progress_signal_completion(); } if (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) { - if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)) { + if (!(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE)) { /* If op is piggybacked with both LOCK and UNLOCK, we only send LOCK ACK back, do not send FLUSH (UNLOCK) ACK. */ mpi_errno = MPIDI_CH3I_Send_flush_ack_pkt(vc, win_ptr, diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index 79dc318..302e167 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -238,7 +238,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_accum_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -595,7 +596,8 @@ int MPIDI_CH3_ReqHandler_GetDerivedDTRecvComplete( MPIDI_VC_t *vc, get_resp_pkt->target_rank = win_ptr->comm_ptr->rank; get_resp_pkt->source_win_handle = rreq->dev.source_win_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (rreq->dev.flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -831,7 +833,6 @@ static int create_derived_datatype(MPID_Request *req, MPID_Datatype **dtp) static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_entry_t *lock_entry) { MPIDI_CH3_Pkt_put_t *put_pkt = &((lock_entry->pkt).put); - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; if (lock_entry->data == NULL) { @@ -846,11 +847,8 @@ static inline int perform_put_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, put_pkt->origin_rank, &vc); - /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_PUT, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_PUT, put_pkt->flags, put_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -866,7 +864,6 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = &upkt.get_resp; MPIDI_CH3_Pkt_get_t *get_pkt = &((lock_entry->pkt).get); MPID_Request *sreq = NULL; - MPIDI_VC_t *vc = NULL; MPI_Aint type_size; size_t len; int iovcnt; @@ -894,7 +891,8 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -938,10 +936,7 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en iovcnt = 2; } - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, get_pkt->origin_rank, &vc); - - mpi_errno = MPIDI_CH3_iSendv(vc, sreq, iov, iovcnt); + mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt); if (mpi_errno != MPI_SUCCESS) { MPID_Request_release(sreq); MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -957,7 +952,6 @@ static inline int perform_get_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_entry_t *lock_entry) { MPIDI_CH3_Pkt_accum_t *acc_pkt = &((lock_entry->pkt).accum); - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; MPIU_Assert(lock_entry->all_data_recved == 1); @@ -980,10 +974,7 @@ static inline int perform_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (win_ptr->shm_allocated == TRUE) MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, acc_pkt->origin_rank, &vc); - - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_ACCUMULATE, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_ACCUMULATE, acc_pkt->flags, acc_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1000,7 +991,6 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &upkt.get_accum_resp; MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt = &((lock_entry->pkt).get_accum); MPID_Request *sreq = NULL; - MPIDI_VC_t *vc = NULL; MPI_Aint type_size; size_t len; int iovcnt; @@ -1065,7 +1055,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP); get_accum_resp_pkt->request_handle = get_accum_pkt->request_handle; get_accum_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1108,10 +1099,7 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc iovcnt = 2; } - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, get_accum_pkt->origin_rank, &vc); - - mpi_errno = MPIDI_CH3_iSendv(vc, sreq, iov, iovcnt); + mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt); if (mpi_errno != MPI_SUCCESS) { MPID_Request_release(sreq); MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -1130,21 +1118,18 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp; MPIDI_CH3_Pkt_fop_t *fop_pkt = &((lock_entry->pkt).fop); MPID_Request *resp_req = NULL; - MPIDI_VC_t *vc = NULL; int mpi_errno = MPI_SUCCESS; /* FIXME: this function is same with PktHandler_FOP(), should do code refactoring on both of them. */ - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, fop_pkt->origin_rank, &vc); - MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP); fop_resp_pkt->request_handle = fop_pkt->request_handle; fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1170,9 +1155,9 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); /* send back the original data */ - MPIU_THREAD_CS_ENTER(CH3COMM,vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); - MPIU_THREAD_CS_EXIT(CH3COMM,vc); + MPIU_THREAD_CS_ENTER(CH3COMM,lock_entry->vc); + mpi_errno = MPIDI_CH3_iStartMsg(lock_entry->vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); + MPIU_THREAD_CS_EXIT(CH3COMM,lock_entry->vc); MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); if (resp_req != NULL) { @@ -1196,7 +1181,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_FOP, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_FOP, fop_pkt->flags, fop_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1213,19 +1198,16 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3_Pkt_cas_resp_t *cas_resp_pkt = &upkt.cas_resp; MPIDI_CH3_Pkt_cas_t *cas_pkt = &((lock_entry->pkt).cas); MPID_Request *send_req = NULL; - MPIDI_VC_t *vc = NULL; MPI_Aint len; int mpi_errno = MPI_SUCCESS; - /* get vc object */ - MPIDI_Comm_get_vc(win_ptr->comm_ptr, cas_pkt->origin_rank, &vc); - MPIDI_Pkt_init(cas_resp_pkt, MPIDI_CH3_PKT_CAS_RESP); cas_resp_pkt->request_handle = cas_pkt->request_handle; cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) cas_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1249,9 +1231,9 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); /* Send the response packet */ - MPIU_THREAD_CS_ENTER(CH3COMM, vc); - mpi_errno = MPIDI_CH3_iStartMsg(vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &send_req); - MPIU_THREAD_CS_EXIT(CH3COMM, vc); + MPIU_THREAD_CS_ENTER(CH3COMM, lock_entry->vc); + mpi_errno = MPIDI_CH3_iStartMsg(lock_entry->vc, cas_resp_pkt, sizeof(*cas_resp_pkt), &send_req); + MPIU_THREAD_CS_EXIT(CH3COMM, lock_entry->vc); MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); @@ -1275,7 +1257,7 @@ static inline int perform_cas_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en } /* do final action */ - mpi_errno = finish_op_on_target(win_ptr, vc, MPIDI_CH3_PKT_CAS, + mpi_errno = finish_op_on_target(win_ptr, lock_entry->vc, MPIDI_CH3_PKT_CAS, cas_pkt->flags, cas_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1295,16 +1277,17 @@ static inline int perform_op_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_ent /* single LOCK request */ MPIDI_CH3_Pkt_lock_t *lock_pkt = &(lock_entry->pkt.lock); - if (lock_pkt->origin_rank == win_ptr->comm_ptr->rank) { - mpi_errno = handle_lock_ack(win_ptr, lock_pkt->origin_rank, + MPIDI_VC_t *my_vc = NULL; + + MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &my_vc); + + if (lock_entry->vc == my_vc) { + mpi_errno = handle_lock_ack(win_ptr, win_ptr->comm_ptr->rank, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } else { - MPIDI_VC_t *vc = NULL; - MPIDI_Comm_get_vc_set_active(win_ptr->comm_ptr, - lock_pkt->origin_rank, &vc); - mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, + mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(lock_entry->vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, lock_pkt->source_win_handle); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); @@ -1410,7 +1393,14 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr) lock_entry_next = lock_entry->next; if (lock_entry->all_data_recved) { - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(lock_entry->pkt, requested_lock, mpi_errno); + MPIDI_CH3_Pkt_flags_t flags; + MPIDI_CH3_PKT_RMA_GET_FLAGS(lock_entry->pkt, flags, mpi_errno); + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) + requested_lock = MPI_LOCK_SHARED; + else { + MPIU_Assert(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + requested_lock = MPI_LOCK_EXCLUSIVE; + } if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, requested_lock) == 1) { /* dequeue entry from lock queue */ MPL_LL_DELETE(win_ptr->lock_queue, win_ptr->lock_queue_tail, lock_entry); @@ -1456,6 +1446,7 @@ int MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete( MPIDI_VC_t *vc, int requested_lock; MPI_Win target_win_handle; MPID_Win *win_ptr = NULL; + MPIDI_CH3_Pkt_flags_t flags; MPIDI_RMA_Lock_entry_t *lock_queue_entry = rreq->dev.lock_queue_entry; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_PIGGYBACKLOCKOPRECVCOMPLETE); @@ -1473,10 +1464,18 @@ int MPIDI_CH3_ReqHandler_PiggybackLockOpRecvComplete( MPIDI_VC_t *vc, lock_queue_entry->all_data_recved = 1; /* try to acquire the lock here */ - MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(lock_queue_entry->pkt, requested_lock, mpi_errno); + MPIDI_CH3_PKT_RMA_GET_FLAGS(lock_queue_entry->pkt, flags, mpi_errno); MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(lock_queue_entry->pkt, target_win_handle, mpi_errno); MPID_Win_get_ptr(target_win_handle, win_ptr); + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) { + requested_lock = MPI_LOCK_SHARED; + } + else { + MPIU_Assert(flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + requested_lock = MPI_LOCK_EXCLUSIVE; + } + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, requested_lock) == 1) { /* dequeue entry from lock queue */ MPL_LL_DELETE(win_ptr->lock_queue, win_ptr->lock_queue_tail, lock_queue_entry); diff --git a/src/mpid/ch3/src/ch3u_rma_oplist.c b/src/mpid/ch3/src/ch3u_rma_oplist.c index 4261903..92b4961 100644 --- a/src/mpid/ch3/src/ch3u_rma_oplist.c +++ b/src/mpid/ch3/src/ch3u_rma_oplist.c @@ -328,7 +328,12 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe /* piggyback on first OP. */ if (target->access_state == MPIDI_RMA_LOCK_CALLED) { MPIU_Assert(curr_op->piggyback_lock_candidate); - flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK; + if (target->lock_type == MPI_LOCK_SHARED) + flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED; + else { + MPIU_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE); + flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE; + } target->access_state = MPIDI_RMA_LOCK_ISSUED; } first_op = 0; @@ -366,7 +371,8 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t *targe PUT/ACC operation. */ } - if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) { + if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) { /* If this operation is piggybacked with LOCK, do not move it out of pending list, and do not complete the user request, because we diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index f05d110..45df32b 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -138,7 +138,6 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; put_pkt->source_win_handle = win_ptr->handle; put_pkt->immed_len = 0; - put_pkt->origin_rank = rank; put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; /* FIXME: For contig and very short operations, use a streamlined op */ @@ -305,7 +304,6 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype get_pkt->dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; - get_pkt->origin_rank = rank; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; /* FIXME: For contig and very short operations, use a streamlined op */ @@ -471,7 +469,6 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; accum_pkt->source_win_handle = win_ptr->handle; accum_pkt->immed_len = 0; - accum_pkt->origin_rank = rank; accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; @@ -645,7 +642,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_pkt->dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; - get_pkt->origin_rank = rank; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = result_addr; @@ -682,7 +678,6 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count, get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_accum_pkt->source_win_handle = win_ptr->handle; get_accum_pkt->immed_len = 0; - get_accum_pkt->origin_rank = rank; get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; @@ -955,7 +950,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, cas_pkt->datatype = datatype; cas_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; cas_pkt->source_win_handle = win_ptr->handle; - cas_pkt->origin_rank = rank; cas_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; @@ -1083,7 +1077,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, get_pkt->dataloop_size = 0; get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; get_pkt->source_win_handle = win_ptr->handle; - get_pkt->origin_rank = rank; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = result_addr; @@ -1103,7 +1096,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, fop_pkt->source_win_handle = win_ptr->handle; fop_pkt->target_win_handle = win_ptr->all_win_handles[target_rank]; fop_pkt->immed_len = 0; - fop_pkt->origin_rank = rank; fop_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; new_ptr->origin_addr = (void *) origin_addr; diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index a64e97c..baa19a6 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -406,7 +406,8 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP); get_resp_pkt->request_handle = get_pkt->request_handle; get_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) get_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -927,7 +928,8 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, cas_resp_pkt->source_win_handle = cas_pkt->source_win_handle; cas_resp_pkt->target_rank = win_ptr->comm_ptr->rank; cas_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) cas_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (cas_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1091,7 +1093,8 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, fop_resp_pkt->source_win_handle = fop_pkt->source_win_handle; fop_resp_pkt->target_rank = win_ptr->comm_ptr->rank; fop_resp_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; - if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK) + if (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED || + fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED; if ((fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH) || (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)) @@ -1315,6 +1318,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPIDI_CH3_Pkt_lock_t *lock_pkt = &pkt->lock; MPID_Win *win_ptr = NULL; + int lock_type; int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK); @@ -1328,7 +1332,14 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr); - if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_pkt->lock_type) == 1) { + if (lock_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED) + lock_type = MPI_LOCK_SHARED; + else { + MPIU_Assert(lock_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE); + lock_type = MPI_LOCK_EXCLUSIVE; + } + + if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 1) { /* send lock granted packet. */ mpi_errno = MPIDI_CH3I_Send_lock_ack_pkt(vc, win_ptr, MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED, lock_pkt->source_win_handle); http://git.mpich.org/mpich.git/commitdiff/d46b848a7e574a92521abe02bb63ba927… commit d46b848a7e574a92521abe02bb63ba9275512d14 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 12:04:46 2015 -0800 Add comments about RMA packet wrappers. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 8a062f4..852160b 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -194,6 +194,8 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_) \ { \ + /* This macro returns target_datatype in RMA operation \ + packets. (PUT, GET, ACC, GACC, CAS, FOP) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -221,6 +223,8 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_) \ { \ + /* This macro returns target_count in RMA operation \ + packets. (PUT, GET, ACC, GACC, CAS, FOP) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -246,6 +250,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_IMMED_LEN(pkt_, immed_len_, err_) \ { \ + /* This macro returns immed_len in RMA operation \ + packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ + packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -284,6 +291,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_) \ { \ + /* This macro returns pointer to immed data in RMA operation \ + packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ + packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -322,6 +332,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(pkt_, lock_type_, err_) \ { \ + /* This macro returns lock type in RMA operation \ + packets (PUT, GET, ACC, GACC, FOP, CAS) and RMA control \ + packets (LOCK). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -352,6 +365,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK(pkt_, origin_rank_, err_) \ { \ + /* This macro returns origin rank (used in acquiring lock) in \ + RMA operation packets (PUT, GET, ACC, GACC, FOP, CAS) and \ + RMA control packets (LOCK). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -382,6 +398,11 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_) \ { \ + /* This macro returns flags in RMA operation packets (PUT, GET, \ + ACC, GACC, FOP, CAS), RMA operation response packets \ + (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control \ + packets (UNLOCK) and RMA control response packets (LOCK_ACK, \ + LOCK_OP_ACK) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -430,6 +451,11 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_) \ { \ + /* This macro erases flags in RMA operation packets (PUT, GET, \ + ACC, GACC, FOP, CAS), RMA operation response packets \ + (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control \ + packets (UNLOCK) and RMA control response packets (LOCK_ACK, \ + LOCK_OP_ACK) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -478,6 +504,11 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_) \ { \ + /* This macro returns source_win_handle in RMA operation \ + packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation \ + response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP), \ + RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control \ + response packets (LOCK_ACK, LOCK_OP_ACK, FLUSH_ACK). */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -535,6 +566,9 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_) \ { \ + /* This macro returns target_win_handle in RMA operation \ + packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control \ + packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ @@ -574,6 +608,8 @@ MPIDI_CH3_PKT_DEFS #define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \ { \ + /* This macro sets dataloop_size in RMA operation packets \ + (PUT, GET, ACC, GACC) */ \ err_ = MPI_SUCCESS; \ switch((pkt_).type) { \ case (MPIDI_CH3_PKT_PUT): \ http://git.mpich.org/mpich.git/commitdiff/064e60ce4e6903c56c27db5d6d8790f4b… commit 064e60ce4e6903c56c27db5d6d8790f4be05f491 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Tue Feb 10 12:04:25 2015 -0800 Modify packet wrappers to make them complete. Some packet wrappers did not include all packet types, this patch adds missed packet types to those wrappers. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index a3e1793..8a062f4 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -261,7 +261,21 @@ MPIDI_CH3_PKT_DEFS immed_len_ = (pkt_).fop.immed_len; \ break; \ case (MPIDI_CH3_PKT_CAS): \ - /* FIXME: we should deal with CAS here */ \ + /* Note that here we return size of origin data, not \ + size of compare data. */ \ + immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ + break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + immed_len_ = (pkt_).get_resp.immed_len; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + immed_len_ = (pkt_).get_accum_resp.immed_len; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + immed_len_ = (pkt_).fop_resp.immed_len; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + immed_len_ = sizeof(MPIDI_CH3_CAS_Immed_u); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ @@ -285,7 +299,21 @@ MPIDI_CH3_PKT_DEFS immed_data_ = (pkt_).fop.data; \ break; \ case (MPIDI_CH3_PKT_CAS): \ - /* FIXME: we should deal with CAS here */ \ + /* Note that here we return pointer of origin data, not \ + pointer of compare data. */ \ + immed_data_ = &((pkt_).cas.origin_data); \ + break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + immed_data_ = (pkt_).get_resp.data; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + immed_data_ = (pkt_).get_accum_resp.data; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + immed_data_ = (pkt_).fop_resp.data; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + immed_data_ = &((pkt_).cas_resp.data); \ break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ @@ -374,6 +402,27 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_FOP): \ flags_ = (pkt_).fop.flags; \ break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + flags_ = (pkt_).get_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + flags_ = (pkt_).get_accum_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + flags_ = (pkt_).fop_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + flags_ = (pkt_).cas_resp.flags; \ + break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + flags_ = (pkt_).unlock.flags; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + flags_ = (pkt_).lock_ack.flags; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + flags_ = (pkt_).lock_op_ack.flags; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ @@ -401,6 +450,27 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_FOP): \ (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE; \ break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + (pkt_).get_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + (pkt_).get_accum_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + (pkt_).fop_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + (pkt_).cas_resp.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + (pkt_).unlock.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + (pkt_).lock_ack.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + (pkt_).lock_op_ack.flags = MPIDI_CH3_PKT_FLAG_NONE; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ @@ -428,9 +498,36 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_FOP): \ win_hdl_ = (pkt_).fop.source_win_handle; \ break; \ + case (MPIDI_CH3_PKT_GET_RESP): \ + win_hdl_ = (pkt_).get_resp.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ + win_hdl_ = (pkt_).get_accum_resp.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FOP_RESP): \ + win_hdl_ = (pkt_).fop_resp.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_CAS_RESP): \ + win_hdl_ = (pkt_).cas_resp.source_win_handle; \ + break; \ case (MPIDI_CH3_PKT_LOCK): \ win_hdl_ = (pkt_).lock.source_win_handle; \ break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + win_hdl_ = (pkt_).unlock.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FLUSH): \ + win_hdl_ = (pkt_).flush.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_ACK): \ + win_hdl_ = (pkt_).lock_ack.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ + win_hdl_ = (pkt_).lock_op_ack.source_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FLUSH_ACK): \ + win_hdl_ = (pkt_).flush_ack.source_win_handle; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ @@ -461,6 +558,15 @@ MPIDI_CH3_PKT_DEFS case (MPIDI_CH3_PKT_LOCK): \ win_hdl_ = (pkt_).lock.target_win_handle; \ break; \ + case (MPIDI_CH3_PKT_UNLOCK): \ + win_hdl_ = (pkt_).unlock.target_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_FLUSH): \ + win_hdl_ = (pkt_).flush.target_win_handle; \ + break; \ + case (MPIDI_CH3_PKT_DECR_AT_COUNTER): \ + win_hdl_ = (pkt_).decr_at_cnt.target_win_handle; \ + break; \ default: \ MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ } \ http://git.mpich.org/mpich.git/commitdiff/fa95883370a01f71aeabeee548e407688… commit fa95883370a01f71aeabeee548e4076880ecbd86 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Mon Feb 9 16:44:25 2015 -0800 Re-apply modifications on mpidpkt.h. This patch re-apply modifications on mpidpkt.h that is temporarily reverted in bb3f962. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 3e0315a..a3e1793 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -86,7 +86,8 @@ typedef enum { MPIDI_CH3_PKT_FOP_RESP, MPIDI_CH3_PKT_CAS_RESP, MPIDI_CH3_PKT_LOCK, - MPIDI_CH3_PKT_LOCK_GRANTED, + MPIDI_CH3_PKT_LOCK_ACK, + MPIDI_CH3_PKT_LOCK_OP_ACK, MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, MPIDI_CH3_PKT_FLUSH_ACK, @@ -502,7 +503,7 @@ typedef struct MPIDI_CH3_Pkt_put { * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; - size_t immed_len; + int immed_len; int lock_type; /* used when piggybacking LOCK message. */ int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_put_t; @@ -533,6 +534,9 @@ typedef struct MPIDI_CH3_Pkt_get_resp { int target_rank; MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; + /* Followings are to piggyback IMMED data */ + int immed_len; + char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_resp_t; typedef struct MPIDI_CH3_Pkt_accum { @@ -551,7 +555,7 @@ typedef struct MPIDI_CH3_Pkt_accum { * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; - size_t immed_len; + int immed_len; int lock_type; /* used when piggybacking LOCK message. */ int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_accum_t; @@ -573,7 +577,7 @@ typedef struct MPIDI_CH3_Pkt_get_accum { * epoch in the case of passive target rma * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; - size_t immed_len; + int immed_len; int lock_type; /* used when piggybacking LOCK message. */ int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_accum_t; @@ -585,6 +589,9 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { int target_rank; MPI_Win source_win_handle; MPIDI_CH3_Pkt_flags_t flags; + /* Followings are to piggyback IMMED data */ + int immed_len; + char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_cas { @@ -648,39 +655,36 @@ typedef struct MPIDI_CH3_Pkt_lock { int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int target_rank; /* Used in unluck/flush response to look up the - * target state at the origin. */ int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { MPIDI_CH3_Pkt_type_t type; - int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int target_rank; /* Used in unluck/flush response to look up the - * target state at the origin. */ - int origin_rank; MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_unlock_t; typedef struct MPIDI_CH3_Pkt_flush { MPIDI_CH3_Pkt_type_t type; - int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int target_rank; /* Used in unluck/flush response to look up the - * target state at the origin. */ - int origin_rank; } MPIDI_CH3_Pkt_flush_t; -typedef struct MPIDI_CH3_Pkt_lock_granted { +typedef struct MPIDI_CH3_Pkt_lock_ack { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win source_win_handle; int target_rank; /* Used in flush_ack response to look up the * target state at the origin. */ -} MPIDI_CH3_Pkt_lock_granted_t; +} MPIDI_CH3_Pkt_lock_ack_t; + +typedef struct MPIDI_CH3_Pkt_lock_op_ack { + MPIDI_CH3_Pkt_type_t type; + MPIDI_CH3_Pkt_flags_t flags; + MPI_Win source_win_handle; + int target_rank; +} MPIDI_CH3_Pkt_lock_op_ack_t; typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; @@ -725,7 +729,8 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_accum_t accum; MPIDI_CH3_Pkt_get_accum_t get_accum; MPIDI_CH3_Pkt_lock_t lock; - MPIDI_CH3_Pkt_lock_granted_t lock_granted; + MPIDI_CH3_Pkt_lock_ack_t lock_ack; + MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack; MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; MPIDI_CH3_Pkt_flush_ack_t flush_ack; http://git.mpich.org/mpich.git/commitdiff/2cbc9180045dea815a7aeb7a7d1fe7f29… commit 2cbc9180045dea815a7aeb7a7d1fe7f299020f72 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Mon Feb 9 16:43:23 2015 -0800 Revert "Code-refactor: arrange RMA pkt structure." This reverts commit 389aab1673b9e54b9172985f69931bf1d731f3d9. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index ed6f4f4..3e0315a 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -86,11 +86,11 @@ typedef enum { MPIDI_CH3_PKT_FOP_RESP, MPIDI_CH3_PKT_CAS_RESP, MPIDI_CH3_PKT_LOCK, + MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, - MPIDI_CH3_PKT_DECR_AT_COUNTER, - MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_FLUSH_ACK, + MPIDI_CH3_PKT_DECR_AT_COUNTER, /* RMA Packets end here */ MPIDI_CH3_PKT_FLOW_CNTL_UPDATE, /* FIXME: Unused */ MPIDI_CH3_PKT_CLOSE, @@ -486,236 +486,214 @@ MPIDI_CH3_PKT_DEFS } \ } -/* RMA packets start here */ - -/********************************************************************************/ -/* RMA packet (from origin to target, including PUT, GET, ACC, GACC, CAS, FOP) */ -/********************************************************************************/ typedef struct MPIDI_CH3_Pkt_put { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ - size_t immed_len; + int dataloop_size; /* for derived datatypes */ + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; + size_t immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_put_t; typedef struct MPIDI_CH3_Pkt_get { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Following is to complete request at origin */ + int dataloop_size; /* for derived datatypes */ MPI_Request request_handle; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_t; +typedef struct MPIDI_CH3_Pkt_get_resp { + MPIDI_CH3_Pkt_type_t type; + MPI_Request request_handle; + /* followings are used to decrement ack_counter at origin */ + int target_rank; + MPI_Win source_win_handle; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_get_resp_t; + typedef struct MPIDI_CH3_Pkt_accum { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Following is to specify ACC op */ + int dataloop_size; /* for derived datatypes */ MPI_Op op; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ - size_t immed_len; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; + size_t immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_accum_t; typedef struct MPIDI_CH3_Pkt_get_accum { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win target_win_handle; - MPI_Win source_win_handle; - /* Followings are to describe target data */ + MPI_Request request_handle; /* For get_accumulate response */ void *addr; int count; MPI_Datatype datatype; - int dataloop_size; - /* Following is to describe ACC op */ + int dataloop_size; /* for derived datatypes */ MPI_Op op; - /* Following is to complete request on origin */ - MPI_Request request_handle; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggback IMMED data */ - size_t immed_len; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ + MPI_Win source_win_handle; /* Used in the last RMA operation in an + * epoch in the case of passive target rma + * with shared locks. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; + size_t immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_get_accum_t; -typedef struct MPIDI_CH3_Pkt_fop { +typedef struct MPIDI_CH3_Pkt_get_accum_resp { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - MPI_Win source_win_handle; - MPI_Win target_win_handle; - /* Followings are to describe target data */ - void *addr; - MPI_Datatype datatype; - /* Following is to speicfy ACC op */ - MPI_Op op; - /* Following is to complete request at origin */ MPI_Request request_handle; - /* Followings are to piggyback IMMED data */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; -} MPIDI_CH3_Pkt_fop_t; + /* followings are used to decrement ack_counter at origin */ + int target_rank; + MPI_Win source_win_handle; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_cas { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - MPI_Win source_win_handle; - MPI_Win target_win_handle; - /* Followings are to describe target data */ - void *addr; MPI_Datatype datatype; - /* Following is to complete request on origin */ + void *addr; MPI_Request request_handle; - /* Followings are to piggyback LOCK */ - int lock_type; - int origin_rank; - /* Followings are to piggyback IMMED data */ + MPI_Win source_win_handle; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ MPIDI_CH3_CAS_Immed_u origin_data; MPIDI_CH3_CAS_Immed_u compare_data; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ } MPIDI_CH3_Pkt_cas_t; - -/*********************************************************************************/ -/* RMA response packet (from target to origin, including GET_RESP, GET_ACC_RESP, */ -/* CAS_RESP, FOP_RESP) */ -/*********************************************************************************/ - -typedef struct MPIDI_CH3_Pkt_get_resp { +typedef struct MPIDI_CH3_Pkt_cas_resp { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ MPI_Request request_handle; - /* TODO: we should add IMMED data here */ - /* Followings are used to decrement ack_counter at origin */ - MPI_Win source_win_handle; + MPIDI_CH3_CAS_Immed_u data; + /* followings are used to decrement ack_counter at orign */ int target_rank; -} MPIDI_CH3_Pkt_get_resp_t; - -typedef struct MPIDI_CH3_Pkt_get_accum_resp { - MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ - MPI_Request request_handle; - /* TODO: we should add IMMED data here */ - /* Followings are used to decrement ack_counter at origin */ MPI_Win source_win_handle; - int target_rank; -} MPIDI_CH3_Pkt_get_accum_resp_t; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_cas_resp_t; -typedef struct MPIDI_CH3_Pkt_fop_resp { +typedef struct MPIDI_CH3_Pkt_fop { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ + MPI_Datatype datatype; + void *addr; + MPI_Op op; MPI_Request request_handle; - /* Followings are used to decrement ack_counter at orign */ MPI_Win source_win_handle; - int target_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; + MPI_Win target_win_handle; /* Used in the last RMA operation in each + * epoch for decrementing rma op counter in + * active target rma and for unlocking window + * in passive target rma. Otherwise set to NULL*/ char data[MPIDI_RMA_IMMED_BYTES]; -} MPIDI_CH3_Pkt_fop_resp_t; + int immed_len; + int lock_type; /* used when piggybacking LOCK message. */ + int origin_rank; /* used when piggybacking LOCK message. */ +} MPIDI_CH3_Pkt_fop_t; -typedef struct MPIDI_CH3_Pkt_cas_resp { +typedef struct MPIDI_CH3_Pkt_fop_resp { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - /* Following is to complete request at origin */ MPI_Request request_handle; - /* Followings are used to decrement ack_counter at orign */ - MPI_Win source_win_handle; + char data[MPIDI_RMA_IMMED_BYTES]; + int immed_len; + /* followings are used to decrement ack_counter at orign */ int target_rank; - /* Following is to piggyback IMMED data */ - MPIDI_CH3_CAS_Immed_u data; -} MPIDI_CH3_Pkt_cas_resp_t; - -/*********************************************************************************/ -/* RMA control packet (from origin to target, including LOCK, UNLOCK, FLUSH) */ -/*********************************************************************************/ + MPI_Win source_win_handle; + MPIDI_CH3_Pkt_flags_t flags; +} MPIDI_CH3_Pkt_fop_resp_t; typedef struct MPIDI_CH3_Pkt_lock { MPIDI_CH3_Pkt_type_t type; + int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int lock_type; - int target_rank; + int target_rank; /* Used in unluck/flush response to look up the + * target state at the origin. */ int origin_rank; } MPIDI_CH3_Pkt_lock_t; typedef struct MPIDI_CH3_Pkt_unlock { MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; + int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int lock_type; - int target_rank; + int target_rank; /* Used in unluck/flush response to look up the + * target state at the origin. */ int origin_rank; + MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_unlock_t; typedef struct MPIDI_CH3_Pkt_flush { MPIDI_CH3_Pkt_type_t type; + int lock_type; MPI_Win target_win_handle; MPI_Win source_win_handle; - int lock_type; - int target_rank; + int target_rank; /* Used in unluck/flush response to look up the + * target state at the origin. */ int origin_rank; } MPIDI_CH3_Pkt_flush_t; -typedef struct MPIDI_CH3_Pkt_decr_at_counter { - MPIDI_CH3_Pkt_type_t type; - MPI_Win target_win_handle; -} MPIDI_CH3_Pkt_decr_at_counter_t; - -/*********************************************************************************/ -/* RMA control response packet (from target to origin, including LOCK_GRANTED, */ -/* FLUSH_ACK) */ -/*********************************************************************************/ - typedef struct MPIDI_CH3_Pkt_lock_granted { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win source_win_handle; - int target_rank; + int target_rank; /* Used in flush_ack response to look up the + * target state at the origin. */ } MPIDI_CH3_Pkt_lock_granted_t; typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; MPI_Win source_win_handle; - int target_rank; + int target_rank; /* Used in flush_ack response to look up the + * target state at the origin. */ + MPIDI_CH3_Pkt_flags_t flags; } MPIDI_CH3_Pkt_flush_ack_t; -/* RMA packets end here */ +typedef struct MPIDI_CH3_Pkt_decr_at_counter { + MPIDI_CH3_Pkt_type_t type; + MPI_Win target_win_handle; +} MPIDI_CH3_Pkt_decr_at_counter_t; typedef struct MPIDI_CH3_Pkt_close { MPIDI_CH3_Pkt_type_t type; @@ -741,25 +719,23 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_rndv_send_t rndv_send; MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req; MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp; - /* RMA packets start here */ MPIDI_CH3_Pkt_put_t put; MPIDI_CH3_Pkt_get_t get; + MPIDI_CH3_Pkt_get_resp_t get_resp; MPIDI_CH3_Pkt_accum_t accum; MPIDI_CH3_Pkt_get_accum_t get_accum; - MPIDI_CH3_Pkt_fop_t fop; - MPIDI_CH3_Pkt_cas_t cas; - MPIDI_CH3_Pkt_get_resp_t get_resp; - MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp; - MPIDI_CH3_Pkt_fop_resp_t fop_resp; - MPIDI_CH3_Pkt_cas_resp_t cas_resp; MPIDI_CH3_Pkt_lock_t lock; + MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; - MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; - MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_flush_ack_t flush_ack; - /* RMA packets end here */ + MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; MPIDI_CH3_Pkt_close_t close; + MPIDI_CH3_Pkt_cas_t cas; + MPIDI_CH3_Pkt_cas_resp_t cas_resp; + MPIDI_CH3_Pkt_fop_t fop; + MPIDI_CH3_Pkt_fop_resp_t fop_resp; + MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp; MPIDI_CH3_Pkt_revoke_t revoke; # if defined(MPIDI_CH3_PKT_DECL) MPIDI_CH3_PKT_DECL http://git.mpich.org/mpich.git/commitdiff/bb3f96236aaab668a2129ec4939976588… commit bb3f96236aaab668a2129ec4939976588fce4ebf Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Sun Feb 8 19:20:12 2015 -0800 Temporarily revert commits for src/mpid/ch3/include/mpidpkt.h We are going to revert the commit 389aab1 because it re-ordered the attributes in RMA packet structs in mpidpkt.h and messed up the alignments. This commit temporarily reverts the following commits, which only reverts modification on mpidpkt.h after commit 389aab1. e36203c3, 45afd1fd, 3a05784f, 87acbbbe, b155e7e We will re-apply those modifications after we revert 389aab1. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 385c99f..ed6f4f4 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -89,8 +89,7 @@ typedef enum { MPIDI_CH3_PKT_UNLOCK, MPIDI_CH3_PKT_FLUSH, MPIDI_CH3_PKT_DECR_AT_COUNTER, - MPIDI_CH3_PKT_LOCK_ACK, - MPIDI_CH3_PKT_LOCK_OP_ACK, + MPIDI_CH3_PKT_LOCK_GRANTED, MPIDI_CH3_PKT_FLUSH_ACK, /* RMA Packets end here */ MPIDI_CH3_PKT_FLOW_CNTL_UPDATE, /* FIXME: Unused */ @@ -507,7 +506,7 @@ typedef struct MPIDI_CH3_Pkt_put { int lock_type; int origin_rank; /* Followings are to piggyback IMMED data */ - int immed_len; + size_t immed_len; char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_put_t; @@ -544,7 +543,7 @@ typedef struct MPIDI_CH3_Pkt_accum { int lock_type; int origin_rank; /* Followings are to piggyback IMMED data */ - int immed_len; + size_t immed_len; char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_accum_t; @@ -566,7 +565,7 @@ typedef struct MPIDI_CH3_Pkt_get_accum { int lock_type; int origin_rank; /* Followings are to piggback IMMED data */ - int immed_len; + size_t immed_len; char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_accum_t; @@ -623,9 +622,6 @@ typedef struct MPIDI_CH3_Pkt_get_resp { /* Followings are used to decrement ack_counter at origin */ MPI_Win source_win_handle; int target_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_resp_t; typedef struct MPIDI_CH3_Pkt_get_accum_resp { @@ -637,9 +633,6 @@ typedef struct MPIDI_CH3_Pkt_get_accum_resp { /* Followings are used to decrement ack_counter at origin */ MPI_Win source_win_handle; int target_rank; - /* Followings are to piggyback IMMED data */ - int immed_len; - char data[MPIDI_RMA_IMMED_BYTES]; } MPIDI_CH3_Pkt_get_accum_resp_t; typedef struct MPIDI_CH3_Pkt_fop_resp { @@ -676,6 +669,7 @@ typedef struct MPIDI_CH3_Pkt_lock { MPI_Win target_win_handle; MPI_Win source_win_handle; int lock_type; + int target_rank; int origin_rank; } MPIDI_CH3_Pkt_lock_t; @@ -684,12 +678,18 @@ typedef struct MPIDI_CH3_Pkt_unlock { MPIDI_CH3_Pkt_flags_t flags; MPI_Win target_win_handle; MPI_Win source_win_handle; + int lock_type; + int target_rank; + int origin_rank; } MPIDI_CH3_Pkt_unlock_t; typedef struct MPIDI_CH3_Pkt_flush { MPIDI_CH3_Pkt_type_t type; MPI_Win target_win_handle; MPI_Win source_win_handle; + int lock_type; + int target_rank; + int origin_rank; } MPIDI_CH3_Pkt_flush_t; typedef struct MPIDI_CH3_Pkt_decr_at_counter { @@ -698,23 +698,16 @@ typedef struct MPIDI_CH3_Pkt_decr_at_counter { } MPIDI_CH3_Pkt_decr_at_counter_t; /*********************************************************************************/ -/* RMA control response packet (from target to origin, including LOCK_ACK, */ -/* LOCK_OP_ACK, FLUSH_ACK) */ +/* RMA control response packet (from target to origin, including LOCK_GRANTED, */ +/* FLUSH_ACK) */ /*********************************************************************************/ -typedef struct MPIDI_CH3_Pkt_lock_ack { - MPIDI_CH3_Pkt_type_t type; - MPIDI_CH3_Pkt_flags_t flags; - MPI_Win source_win_handle; - int target_rank; -} MPIDI_CH3_Pkt_lock_ack_t; - -typedef struct MPIDI_CH3_Pkt_lock_op_ack { +typedef struct MPIDI_CH3_Pkt_lock_granted { MPIDI_CH3_Pkt_type_t type; MPIDI_CH3_Pkt_flags_t flags; MPI_Win source_win_handle; int target_rank; -} MPIDI_CH3_Pkt_lock_op_ack_t; +} MPIDI_CH3_Pkt_lock_granted_t; typedef struct MPIDI_CH3_Pkt_flush_ack { MPIDI_CH3_Pkt_type_t type; @@ -763,8 +756,7 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_unlock_t unlock; MPIDI_CH3_Pkt_flush_t flush; MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; - MPIDI_CH3_Pkt_lock_ack_t lock_ack; - MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack; + MPIDI_CH3_Pkt_lock_granted_t lock_granted; MPIDI_CH3_Pkt_flush_ack_t flush_ack; /* RMA packets end here */ MPIDI_CH3_Pkt_close_t close; ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/include/mpid_rma_issue.h | 151 ++------ src/mpid/ch3/include/mpid_rma_lockqueue.h | 3 +- src/mpid/ch3/include/mpid_rma_types.h | 1 + src/mpid/ch3/include/mpidimpl.h | 2 + src/mpid/ch3/include/mpidpkt.h | 613 ++++++++++++++++------------- src/mpid/ch3/include/mpidpre.h | 1 - src/mpid/ch3/include/mpidrma.h | 141 ++++--- src/mpid/ch3/src/ch3u_handle_recv_pkt.c | 18 +- src/mpid/ch3/src/ch3u_handle_recv_req.c | 509 +++++++++++++++++-------- src/mpid/ch3/src/ch3u_handle_send_req.c | 20 +- src/mpid/ch3/src/ch3u_request.c | 1 - src/mpid/ch3/src/ch3u_rma_oplist.c | 18 +- src/mpid/ch3/src/ch3u_rma_ops.c | 460 +++++++++++++++------- src/mpid/ch3/src/ch3u_rma_pkthandler.c | 555 ++++++++++++++++++-------- test/mpi/rma/atomic_rmw_gacc.c | 12 +- 15 files changed, 1574 insertions(+), 931 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-144-gb21dc46
by noreply＠mpich.org 09 Feb '15

09 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via b21dc461115bdf99d42176ecea8e8efc763d8a2d (commit) via 67577443f839be953e7c9c9dfd53c06cbd08525f (commit) from b210805e1049f83f09545cc325b6c667b3ebfc69 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/b21dc461115bdf99d42176ecea8e8efc7… commit b21dc461115bdf99d42176ecea8e8efc763d8a2d Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Mon Feb 9 10:19:48 2015 -0600 require libtool >= 2.4.3 Two libtool patches we were carrying were released in version 2.4.3. This commit drops our patches and bumps version required to run autogen.sh. Our patch for ifort on OSX is still present and updated to work with the new version. Signed-off-by: Wesley Bland <wbland(a)anl.gov> diff --git a/autogen.sh b/autogen.sh index 87c2246..5370dbf 100755 --- a/autogen.sh +++ b/autogen.sh @@ -544,7 +544,7 @@ fi echo_n "Checking for libtool version... " recreate_tmp -ver=2.4 +ver=2.4.3 cat <<EOF >.tmp/configure.ac AC_INIT(testver,1.0) AC_CONFIG_AUX_DIR([m4]) @@ -924,33 +924,6 @@ if [ "$do_build_configure" = "yes" ] ; then # Older versions are not supported to build mpich. # Newer versions should have this patch already included. if [ -f $amdir/confdb/libtool.m4 ] ; then - echo_n "Patching libtool.m4 to enable support for powerpcle... " - powerpcle_patch_requires_rebuild=no - patch -N -s -l $amdir/confdb/libtool.m4 maint/0001-libtool-powerpc-le-linux-support.patch - if [ $? -eq 0 ] ; then - powerpcle_patch_requires_rebuild=yes - # Remove possible leftovers, which don't imply a failure - rm -f $amdir/confdb/libtool.m4.orig - echo "done" - else - echo "failed" - fi - - # There is no need to patch if we're not going to use Fortran. - nagfor_patch_requires_rebuild=no - if [ $do_bindings = "yes" ] ; then - echo_n "Patching libtool.m4 for compatibility with nagfor shared libraries... " - patch -N -s -l $amdir/confdb/libtool.m4 maint/libtool.m4.patch - if [ $? -eq 0 ] ; then - nagfor_patch_requires_rebuild=yes - # Remove possible leftovers, which don't imply a failure - rm -f $amdir/confdb/libtool.m4.orig - echo "done" - else - echo "failed" - fi - fi - # There is no need to patch if we're not going to use Fortran. ifort_patch_requires_rebuild=no if [ $do_bindings = "yes" ] ; then @@ -966,7 +939,7 @@ if [ "$do_build_configure" = "yes" ] ; then fi fi - if [ $powerpcle_patch_requires_rebuild = "yes" -o $nagfor_patch_requires_rebuild = "yes" -o $ifort_patch_requires_rebuild = "yes" ] ; then + if [ $ifort_patch_requires_rebuild = "yes" ] ; then # Rebuild configure (cd $amdir && $autoconf -f) || exit 1 # Reset libtool.m4 timestamps to avoid confusing make diff --git a/maint/0001-libtool-powerpc-le-linux-support.patch b/maint/0001-libtool-powerpc-le-linux-support.patch deleted file mode 100644 index 52e122a..0000000 --- a/maint/0001-libtool-powerpc-le-linux-support.patch +++ /dev/null @@ -1,111 +0,0 @@ -From 723f678b8f297544b64880c8d80b60328e82cb45 Mon Sep 17 00:00:00 2001 -From: Alan Modra <amodra(a)bigpond.net.au> -Date: Thu, 6 Jun 2013 14:48:22 +0930 -Subject: [PATCH] libtool: powerpc*le-linux support - -This is a combination of 5 commits. - -See libtool version 2.4.2.418 -http://git.savannah.gnu.org/cgit/libtool.git/log/?h=v2.4.2.418 - -========================================================================== - - commit 12bf693d2d317c3313ee91058b2289d65a57f386 - Author: Alan Modra <amodra(a)bigpond.net.au> - Date: Thu Jun 6 14:48:22 2013 +0930 - - libtool: fix refixed unmangled powerpc*le-linux support patch - - * m4/libtool.m4: fix refixed badly unmangled hunks from earlier - powerpc*le changeset. - Reported by Peter Rosin. - - Signed-off-by: Gary V. Vaughan <gary(a)gnu.org> - - commit aa14ead14c5e375789f08026d9ece5963a9322c2 - Author: Alan Modra <amodra(a)bigpond.net.au> - Date: Thu Jun 6 14:48:22 2013 +0930 - - libtool: refix unmangled powerpc*le-linux support patch - - * m4/libtool.m4: refix badly unmangled hunks from earlier - powerpc*le changeset. - Reported by Peter Rosin. - - Signed-off-by: Gary V. Vaughan <gary(a)gnu.org> - - commit bb1c8bca8aee6e487aaf6b320b8f56f6ac0d21ac - Author: Gary V. Vaughan <gary(a)gnu.org> - Date: Thu Aug 22 15:38:00 2013 +0700 - - libtool: refactor powerpc*le-linux case branch expressions. - - libtool (_LT_ENABLE_LOCK): make inner case branch expressions - consistent with outer case expression. - Reported by Peter Rosin. - - Signed-off-by: Gary V. Vaughan <gary(a)gnu.org> - - commit 03754a10041e86b2bd41b404a9ad824ef28bee7e - Author: Alan Modra <amodra(a)bigpond.net.au> - Date: Thu Jun 6 14:48:22 2013 +0930 - - libtool: fix mangled powerpc*le-linux support patch - - * m4/libtool.m4: unmangled badly pasted hunks from previous - changeset. - - commit f21c4d470423ab5b108918eaa5db295f644b12d1 - Author: Alan Modra <amodra(a)bigpond.net.au> - Date: Thu Jun 6 14:48:22 2013 +0930 - - libtool: initial powerpc*le-linux support - - * m4/libtool.m4 (ld -m flags): Remove non-canonical ppc host match. - Support little-endian powerpc linux host. - -========================================================================== ---- - libltdl/m4/libtool.m4 | 12 +++++++++--- - 1 files changed, 9 insertions(+), 3 deletions(-) - -diff --git a/libltdl/m4/libtool.m4 b/libltdl/m4/libtool.m4 -index d812584..7aefebc 100644 ---- a/libltdl/m4/libtool.m4 -+++ b/libltdl/m4/libtool.m4 -@@ -1268,7 +1268,7 @@ ia64-*-hpux*) - rm -rf conftest* - ;; - --x86_64-*kfreebsd*-gnu|x86_64-*linux*|ppc*-*linux*|powerpc*-*linux*| \ -+x86_64-*kfreebsd*-gnu|x86_64-*linux*|powerpc*-*linux*| \ - s390*-*linux*|s390*-*tpf*|sparc*-*linux*) - # Find out which ABI we are using. - echo 'int i;' > conftest.$ac_ext -@@ -1282,7 +1282,10 @@ s390*-*linux*|s390*-*tpf*|sparc*-*linux*) - x86_64-*linux*) - LD="${LD-ld} -m elf_i386" - ;; -- ppc64-*linux*|powerpc64-*linux*) -+ powerpc64le-*linux*) -+ LD="${LD-ld} -m elf32lppclinux" -+ ;; -+ powerpc64-*linux*) - LD="${LD-ld} -m elf32ppclinux" - ;; - s390x-*linux*) -@@ -1301,7 +1304,10 @@ s390*-*linux*|s390*-*tpf*|sparc*-*linux*) - x86_64-*linux*) - LD="${LD-ld} -m elf_x86_64" - ;; -- ppc*-*linux*|powerpc*-*linux*) -+ powerpcle-*linux*) -+ LD="${LD-ld} -m elf64lppc" -+ ;; -+ powerpc-*linux*) - LD="${LD-ld} -m elf64ppc" - ;; - s390*-*linux*|s390*-*tpf*) --- -1.7.1 - diff --git a/maint/darwin-ifort.patch b/maint/darwin-ifort.patch index 42c7816..7ce8b39 100644 --- a/maint/darwin-ifort.patch +++ b/maint/darwin-ifort.patch @@ -1,11 +1,11 @@ --- confdb/libtool.m4~ 2014-12-23 10:59:38.000000000 -0600 +++ confdb/libtool.m4 2014-12-23 11:05:54.000000000 -0600 -@@ -1097,7 +1097,10 @@ +@@ -1088,7 +1088,10 @@ m4_defun([_LT_DARWIN_LINKER_FEATURES], _LT_TAGVAR(link_all_deplibs, $1)=yes - _LT_TAGVAR(allow_undefined_flag, $1)="$_lt_dar_allow_undefined" + _LT_TAGVAR(allow_undefined_flag, $1)=$_lt_dar_allow_undefined case $cc_basename in -- ifort*) _lt_dar_can_shared=yes ;; -+ ifort*) +- ifort*|nagfor*) _lt_dar_can_shared=yes ;; ++ ifort*|nagfor*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _lt_dar_can_shared=yes + ;; diff --git a/maint/libtool.m4.patch b/maint/libtool.m4.patch deleted file mode 100644 index 4ad7628..0000000 --- a/maint/libtool.m4.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- confdb/libtool.m4 2013-03-29 16:26:23.162062517 -0500 -+++ confdb/libtool.m4 2013-06-21 14:22:05.138914413 -0500 -@@ -4817,6 +4817,8 @@ - lf95*) # Lahey Fortran 8.1 - _LT_TAGVAR(whole_archive_flag_spec, $1)= - tmp_sharedflag='--shared' ;; -+ nagfor*) # NAGFOR 5.3 -+ tmp_sharedflag='-Wl,-shared' ;; - xl[[cC]]* | bgxl[[cC]]* | mpixl[[cC]]*) # IBM XL C 8.0 on PPC (deal with xlf below) - tmp_sharedflag='-qmkshrobj' - tmp_addflag= ;; http://git.mpich.org/mpich.git/commitdiff/67577443f839be953e7c9c9dfd53c06cb… commit 67577443f839be953e7c9c9dfd53c06cbd08525f Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Sun Feb 8 20:23:42 2015 -0600 require automake >= 1.15 Fixes a potential "arg list too long" error at make time. See https://lists.gnu.org/archive/html/bug-automake/2014-10/msg00009.html for more info. Closes #2215 Signed-off-by: Wesley Bland <wbland(a)anl.gov> diff --git a/autogen.sh b/autogen.sh index 8a8ccd9..87c2246 100755 --- a/autogen.sh +++ b/autogen.sh @@ -506,7 +506,7 @@ fi echo_n "Checking for automake version... " recreate_tmp -ver=1.12.3 +ver=1.15 cat > .tmp/configure.ac<<EOF AC_INIT(testver,1.0) AC_CONFIG_AUX_DIR([m4]) ----------------------------------------------------------------------- Summary of changes: autogen.sh | 33 +------ maint/0001-libtool-powerpc-le-linux-support.patch | 111 --------------------- maint/darwin-ifort.patch | 8 +- maint/libtool.m4.patch | 11 -- 4 files changed, 7 insertions(+), 156 deletions(-) delete mode 100644 maint/0001-libtool-powerpc-le-linux-support.patch delete mode 100644 maint/libtool.m4.patch hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-142-gb210805
by noreply＠mpich.org 09 Feb '15

09 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via b210805e1049f83f09545cc325b6c667b3ebfc69 (commit) from 51fdbedcf352a56e5176192bfd665820cd64dadf (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/b210805e1049f83f09545cc325b6c667b… commit b210805e1049f83f09545cc325b6c667b3ebfc69 Author: Igor Ivanov <Igor.Ivanov(a)itseez.com> Date: Mon Feb 9 14:36:02 2015 +0200 mpid/sched: Fix issue with schedule entries list processing Call of MPID_Sched_cb callback function can force list memory reallocation. As a result entry point proccessed before call can become invalid. It should be set again after callback call. Signed-off-by: Devendar Bureddy <devendar(a)mellanox.com> Signed-off-by: Igor Ivanov <Igor.Ivanov(a)itseez.com> Signed-off-by: Wesley Bland <wbland(a)anl.gov> diff --git a/src/mpid/common/sched/mpid_sched.c b/src/mpid/common/sched/mpid_sched.c index ecda136..b01a4d3 100644 --- a/src/mpid/common/sched/mpid_sched.c +++ b/src/mpid/common/sched/mpid_sched.c @@ -217,6 +217,8 @@ static int MPIDU_Sched_start_entry(struct MPIDU_Sched *s, size_t idx, struct MPI MPIU_DBG_MSG_D(COMM, VERBOSE, "starting CB entry %d\n", (int) idx); if (e->u.cb.cb_type == MPIDU_SCHED_CB_TYPE_1) { ret_errno = e->u.cb.u.cb_p(r->comm, s->tag, e->u.cb.cb_state); + /* Sched entries list can be reallocated inside callback */ + e = &s->entries[idx]; if (unlikely(ret_errno)) { if (MPIR_ERR_NONE == r->errflag) { if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(ret_errno)) { @@ -232,6 +234,8 @@ static int MPIDU_Sched_start_entry(struct MPIDU_Sched *s, size_t idx, struct MPI } else if (e->u.cb.cb_type == MPIDU_SCHED_CB_TYPE_2) { ret_errno = e->u.cb.u.cb2_p(r->comm, s->tag, e->u.cb.cb_state, e->u.cb.cb_state2); + /* Sched entries list can be reallocated inside callback */ + e = &s->entries[idx]; if (unlikely(ret_errno)) { if (MPIR_ERR_NONE == r->errflag) { if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(ret_errno)) { @@ -281,6 +285,8 @@ static int MPIDU_Sched_continue(struct MPIDU_Sched *s) if (e->status == MPIDU_SCHED_ENTRY_STATUS_NOT_STARTED) { mpi_errno = MPIDU_Sched_start_entry(s, i, e); + /* Sched entries list can be reallocated inside callback */ + e = &s->entries[i]; if (mpi_errno) MPIU_ERR_POP(mpi_errno); } ----------------------------------------------------------------------- Summary of changes: src/mpid/common/sched/mpid_sched.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-141-g51fdbed
by noreply＠mpich.org 08 Feb '15

08 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 51fdbedcf352a56e5176192bfd665820cd64dadf (commit) via bad898f9df13b5060cbf43ee4acdb3b7b4c9a0f7 (commit) via 65bf0d776e8932e7acb26714ee5a8c01ebf004ca (commit) via 8c5cb1e67248e29a7a5e4523b5adcfa044ab8a93 (commit) via 346050ea58448a53995b1581bb82b291f4a2d5f7 (commit) from 7dfe284051cacc8ef79f15e0c01c81b28f9fe78e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/51fdbedcf352a56e5176192bfd665820c… commit 51fdbedcf352a56e5176192bfd665820cd64dadf Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Thu Feb 5 01:41:01 2015 -0800 Add tests to test the atomicity for FOP, CAS and GACC operations. The entire "read-modify-write" should be atomic for CAS, FOP and GACC operations. This patch adds corresponding tests for them. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/test/mpi/rma/Makefile.am b/test/mpi/rma/Makefile.am index e2a3eb3..5c0a5e8 100644 --- a/test/mpi/rma/Makefile.am +++ b/test/mpi/rma/Makefile.am @@ -140,7 +140,10 @@ noinst_PROGRAMS = \ get-struct \ rput_local_comp \ racc_local_comp \ - at_complete + at_complete \ + atomic_rmw_fop \ + atomic_rmw_cas \ + atomic_rmw_gacc if BUILD_MPIX_TESTS noinst_PROGRAMS += aint diff --git a/test/mpi/rma/atomic_rmw_cas.c b/test/mpi/rma/atomic_rmw_cas.c new file mode 100644 index 0000000..2b9a711 --- /dev/null +++ b/test/mpi/rma/atomic_rmw_cas.c @@ -0,0 +1,129 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2015 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +/* This test is going to test the atomicity for "read-modify-write" in CAS + * operations */ + +/* There are three processes involved in this test: P0 (origin_shm), P1 (origin_am), + * and P2 (dest). P0 and P1 issues one CAS to P2 via SHM and AM respectively. + * For P0, origin value is 1 and compare value is 0; for P1, origin value is 0 and + * compare value is 1; for P2, initial target value is 0. The correct results can + * only be one of the following cases: + * + * (1) result value on P0: 0, result value on P1: 0, target value on P2: 1. + * (2) result value on P0: 0, result value on P1: 1, target value on P2: 0. + * + * All other results are not correct. */ + +#include "mpi.h" +#include <stdio.h> + +#define LOOP_SIZE 10000 +#define CHECK_TAG 123 + +int main (int argc, char *argv[]) { + int rank, size, i, j, k; + int errors = 0; + int origin_shm, origin_am, dest; + int *orig_buf = NULL, *result_buf = NULL, *compare_buf = NULL, + *target_buf = NULL, *check_buf = NULL; + MPI_Win win; + MPI_Status status; + + MPI_Init(&argc, &argv); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (size != 3) { + /* run this test with three processes */ + goto exit_test; + } + + /* this works when MPIR_PARAM_CH3_ODD_EVEN_CLIQUES is set */ + dest = 2; + origin_shm = 0; + origin_am = 1; + + if (rank != dest) { + MPI_Alloc_mem(sizeof(int), MPI_INFO_NULL, &orig_buf); + MPI_Alloc_mem(sizeof(int), MPI_INFO_NULL, &result_buf); + MPI_Alloc_mem(sizeof(int), MPI_INFO_NULL, &compare_buf); + } + + MPI_Win_allocate(sizeof(int), sizeof(int), MPI_INFO_NULL, + MPI_COMM_WORLD, &target_buf, &win); + + for (k = 0; k < LOOP_SIZE; k++) { + + /* init buffers */ + if (rank == origin_shm) { + orig_buf[0] = 1; + compare_buf[0] = 0; + result_buf[0] = 0; + } + else if (rank == origin_am) { + orig_buf[0] = 0; + compare_buf[0] = 1; + result_buf[0] = 0; + } + else { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + target_buf[0] = 0; + MPI_Win_unlock(rank, win); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* perform FOP */ + MPI_Win_lock_all(0, win); + if (rank != dest) { + MPI_Compare_and_swap(orig_buf, compare_buf, result_buf, MPI_INT, dest, 0, win); + MPI_Win_flush(dest, win); + } + MPI_Win_unlock_all(win); + + MPI_Barrier(MPI_COMM_WORLD); + + /* check results */ + if (rank != dest) { + MPI_Gather(result_buf, 1, MPI_INT, check_buf, 1, MPI_INT, dest, MPI_COMM_WORLD); + } + else { + MPI_Alloc_mem(sizeof(int) * 3, MPI_INFO_NULL, &check_buf); + MPI_Gather(target_buf, 1, MPI_INT, check_buf, 1, MPI_INT, dest, MPI_COMM_WORLD); + + if (!(check_buf[dest] == 0 && check_buf[origin_shm] == 0 && check_buf[origin_am] == 1) && + !(check_buf[dest] == 1 && check_buf[origin_shm] == 0 && check_buf[origin_am] == 0)) { + + printf("Wrong results: target result = %d, origin_shm result = %d, origin_am result = %d\n", + check_buf[dest], check_buf[origin_shm], check_buf[origin_am]); + + printf("Expected results (1): target result = 1, origin_shm result = 0, origin_am result = 0\n"); + printf("Expected results (2): target result = 0, origin_shm result = 0, origin_am result = 1\n"); + + errors++; + } + + MPI_Free_mem(check_buf); + } + } + + MPI_Win_free(&win); + + if (rank == origin_am || rank == origin_shm) { + MPI_Free_mem(orig_buf); + MPI_Free_mem(result_buf); + MPI_Free_mem(compare_buf); + } + + exit_test: + if (rank == dest && errors == 0) + printf(" No Errors\n"); + + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/rma/atomic_rmw_fop.c b/test/mpi/rma/atomic_rmw_fop.c new file mode 100644 index 0000000..873efe8 --- /dev/null +++ b/test/mpi/rma/atomic_rmw_fop.c @@ -0,0 +1,131 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2015 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +/* This test is going to test the atomicity for "read-modify-write" in FOP + * operations */ + +/* There are three processes involved in this test: P0 (origin_shm), P1 (origin_am), + * and P2 (dest). P0 and P1 issues multiple FOP with MPI_SUM and integer (value 1) + * to P2 via SHM and AM respectively. The correct results should be that the + * results on P0 and P1 never be the same. */ + +#include "mpi.h" +#include <stdio.h> + +#define AM_BUF_SIZE 10 +#define SHM_BUF_SIZE 1000 +#define WIN_BUF_SIZE 1 + +#define LOOP_SIZE 15 +#define CHECK_TAG 123 + +int main (int argc, char *argv[]) { + int rank, size, i, j, k; + int errors = 0, all_errors = 0; + int origin_shm, origin_am, dest; + int my_buf_size; + int *orig_buf = NULL, *result_buf = NULL, *target_buf = NULL, *check_buf = NULL; + MPI_Win win; + MPI_Status status; + + MPI_Init(&argc, &argv); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (size != 3) { + /* run this test with three processes */ + goto exit_test; + } + + /* this works when MPIR_PARAM_CH3_ODD_EVEN_CLIQUES is set */ + dest = 2; + origin_shm = 0; + origin_am = 1; + + if (rank == origin_am) my_buf_size = AM_BUF_SIZE; + else if (rank == origin_shm) my_buf_size = SHM_BUF_SIZE; + + if (rank != dest) { + MPI_Alloc_mem(sizeof(int) * my_buf_size, MPI_INFO_NULL, &orig_buf); + MPI_Alloc_mem(sizeof(int) * my_buf_size, MPI_INFO_NULL, &result_buf); + } + + MPI_Win_allocate(sizeof(int) * WIN_BUF_SIZE, sizeof(int), MPI_INFO_NULL, + MPI_COMM_WORLD, &target_buf, &win); + + for (k = 0; k < LOOP_SIZE; k++) { + + /* init buffers */ + if (rank != dest) { + for (i = 0; i < my_buf_size; i++) {orig_buf[i] = 1; result_buf[i] = 0;} + } + else { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + for (i = 0; i < WIN_BUF_SIZE; i++) {target_buf[i] = 0;} + MPI_Win_unlock(rank, win); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* perform FOP */ + MPI_Win_lock_all(0, win); + if (rank != dest) { + for (i = 0; i < my_buf_size; i++) { + MPI_Fetch_and_op(&(orig_buf[i]), &(result_buf[i]), MPI_INT, dest, 0, MPI_SUM, win); + MPI_Win_flush(dest, win); + } + } + MPI_Win_unlock_all(win); + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank != dest) { + /* check results on P0 and P2 (origin) */ + if (rank == origin_am) { + MPI_Send(result_buf, AM_BUF_SIZE, MPI_INT, origin_shm, CHECK_TAG, MPI_COMM_WORLD); + } + else if (rank == origin_shm) { + MPI_Alloc_mem(sizeof(int) * AM_BUF_SIZE, MPI_INFO_NULL, &check_buf); + MPI_Recv(check_buf, AM_BUF_SIZE, MPI_INT, origin_am, CHECK_TAG, MPI_COMM_WORLD, &status); + for (i = 0; i < AM_BUF_SIZE; i++) { + for (j = 0; j < SHM_BUF_SIZE; j++) { + if (check_buf[i] == result_buf[j]) { + printf("LOOP=%d, rank=%d, FOP, both check_buf[%d] and result_buf[%d] equal to %d, expected to be different. \n", + k, rank, i, j, check_buf[i]); + errors++; + } + } + } + MPI_Free_mem(check_buf); + } + } + else { + /* check results on P1 */ + if (target_buf[0] != AM_BUF_SIZE + SHM_BUF_SIZE) { + printf("LOOP=%d, rank=%d, FOP, target_buf[0] = %d, expected %d. \n", + k, rank, target_buf[0], AM_BUF_SIZE+SHM_BUF_SIZE); + errors++; + } + } + } + + MPI_Win_free(&win); + + if (rank == origin_am || rank == origin_shm) { + MPI_Free_mem(orig_buf); + MPI_Free_mem(result_buf); + } + + exit_test: + MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + + if (rank == 0 && all_errors == 0) + printf(" No Errors\n"); + + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/rma/atomic_rmw_gacc.c b/test/mpi/rma/atomic_rmw_gacc.c new file mode 100644 index 0000000..d04e9c3 --- /dev/null +++ b/test/mpi/rma/atomic_rmw_gacc.c @@ -0,0 +1,240 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2015 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +/* This test is going to test the atomicity for "read-modify-write" in GACC + * operations */ + +/* This test is similiar with atomic_rmw_fop.c. + * There are three processes involved in this test: P0 (origin_shm), P1 (origin_am), + * and P2 (dest). P0 and P1 issues multiple GACC with MPI_SUM and OP_COUNT integers + * (value 1) to P2 via SHM and AM respectively. The correct results should be that the + * results on P0 and P1 never be the same for intergers on the corresponding index + * in [0...OP_COUNT-1]. + */ + +#include "mpi.h" +#include <stdio.h> + +#define OP_COUNT 10 +#define AM_BUF_NUM 10 +#define SHM_BUF_NUM 10000 +#define WIN_BUF_NUM 1 + +#define LOOP_SIZE 15 +#define CHECK_TAG 123 + +int rank, size; +int dest, origin_shm, origin_am; +int *orig_buf = NULL, *result_buf = NULL, *target_buf = NULL, *check_buf = NULL; + +void checkResults(int loop_k, int *errors) { + int i, j, m; + MPI_Status status; + + if (rank != dest) { + /* check results on P0 and P2 (origin) */ + if (rank == origin_am) { + MPI_Send(result_buf, AM_BUF_NUM * OP_COUNT, MPI_INT, origin_shm, CHECK_TAG, MPI_COMM_WORLD); + } + else if (rank == origin_shm) { + MPI_Alloc_mem(sizeof(int) * AM_BUF_NUM * OP_COUNT, MPI_INFO_NULL, &check_buf); + MPI_Recv(check_buf, AM_BUF_NUM * OP_COUNT, MPI_INT, origin_am, CHECK_TAG, MPI_COMM_WORLD, &status); + for (i = 0; i < AM_BUF_NUM; i++) { + for (j = 0; j < SHM_BUF_NUM; j++) { + for (m = 0; m < OP_COUNT; m++) { + if (check_buf[i*OP_COUNT+m] == result_buf[j*OP_COUNT+m]) { + printf("LOOP=%d, rank=%d, FOP, both check_buf[%d] and result_buf[%d] equal to %d, expected to be different. \n", + loop_k, rank, i*OP_COUNT+m, j*OP_COUNT+m, check_buf[i*OP_COUNT+m]); + (*errors)++; + } + } + } + } + MPI_Free_mem(check_buf); + } + } + else { + /* check results on P1 */ + for (i = 0; i < OP_COUNT; i++) { + if (target_buf[i] != AM_BUF_NUM + SHM_BUF_NUM) { + printf("LOOP=%d, rank=%d, FOP, target_buf[%d] = %d, expected %d. \n", + loop_k, rank, i, target_buf[i], AM_BUF_NUM+SHM_BUF_NUM); + (*errors)++; + } + } + } +} + +int main (int argc, char *argv[]) { + int i, j, k; + int errors = 0, all_errors = 0; + int my_buf_num; + MPI_Win win; + MPI_Datatype origin_dtp, target_dtp; + + MPI_Init(&argc, &argv); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (size != 3) { + /* run this test with three processes */ + goto exit_test; + } + + MPI_Type_contiguous(OP_COUNT, MPI_INT, &origin_dtp); + MPI_Type_commit(&origin_dtp); + MPI_Type_contiguous(OP_COUNT, MPI_INT, &target_dtp); + MPI_Type_commit(&target_dtp); + + /* this works when MPIR_PARAM_CH3_ODD_EVEN_CLIQUES is set */ + dest = 2; + origin_shm = 0; + origin_am = 1; + + if (rank == origin_am) my_buf_num = AM_BUF_NUM; + else if (rank == origin_shm) my_buf_num = SHM_BUF_NUM; + + if (rank != dest) { + MPI_Alloc_mem(sizeof(int) * my_buf_num * OP_COUNT, MPI_INFO_NULL, &orig_buf); + MPI_Alloc_mem(sizeof(int) * my_buf_num * OP_COUNT, MPI_INFO_NULL, &result_buf); + } + + MPI_Win_allocate(sizeof(int) * WIN_BUF_NUM * OP_COUNT, sizeof(int), MPI_INFO_NULL, + MPI_COMM_WORLD, &target_buf, &win); + + for (k = 0; k < LOOP_SIZE; k++) { + + /* ====== Part 1: test basic datatypes ======== */ + + /* init buffers */ + if (rank != dest) { + for (i = 0; i < my_buf_num * OP_COUNT; i++) {orig_buf[i] = 1; result_buf[i] = 0;} + } + else { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + for (i = 0; i < WIN_BUF_NUM * OP_COUNT; i++) {target_buf[i] = 0;} + MPI_Win_unlock(rank, win); + } + + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Win_lock_all(0, win); + if (rank != dest) { + for (i = 0; i < my_buf_num; i++) { + MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), OP_COUNT, MPI_INT, + &(result_buf[i*OP_COUNT]), OP_COUNT, MPI_INT, + dest, 0, OP_COUNT, MPI_INT, MPI_SUM, win); + MPI_Win_flush(dest, win); + } + } + MPI_Win_unlock_all(win); + + MPI_Barrier(MPI_COMM_WORLD); + + checkResults(k, &errors); + + /* ====== Part 2: test derived datatypes (origin derived, target derived) ======== */ + + /* init buffers */ + if (rank != dest) { + for (i = 0; i < my_buf_num * OP_COUNT; i++) {orig_buf[i] = 1; result_buf[i] = 0;} + } + else { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + for (i = 0; i < WIN_BUF_NUM * OP_COUNT; i++) {target_buf[i] = 0;} + MPI_Win_unlock(rank, win); + } + + MPI_Win_lock_all(0, win); + if (rank != dest) { + for (i = 0; i < my_buf_num; i++) { + MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), 1, origin_dtp, + &(result_buf[i*OP_COUNT]), 1, origin_dtp, + dest, 0, 1, target_dtp, MPI_SUM, win); + MPI_Win_flush(dest, win); + } + } + MPI_Win_unlock_all(win); + + MPI_Barrier(MPI_COMM_WORLD); + + checkResults(k, &errors); + + /* ====== Part 3: test derived datatypes (origin basic, target derived) ======== */ + + /* init buffers */ + if (rank != dest) { + for (i = 0; i < my_buf_num * OP_COUNT; i++) {orig_buf[i] = 1; result_buf[i] = 0;} + } + else { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + for (i = 0; i < WIN_BUF_NUM * OP_COUNT; i++) {target_buf[i] = 0;} + MPI_Win_unlock(rank, win); + } + + MPI_Win_lock_all(0, win); + if (rank != dest) { + for (i = 0; i < my_buf_num; i++) { + MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), OP_COUNT, MPI_INT, + &(result_buf[i*OP_COUNT]), OP_COUNT, MPI_INT, + dest, 0, 1, target_dtp, MPI_SUM, win); + MPI_Win_flush(dest, win); + } + } + MPI_Win_unlock_all(win); + + MPI_Barrier(MPI_COMM_WORLD); + + checkResults(k, &errors); + + /* ====== Part 4: test derived datatypes (origin derived target basic) ======== */ + + /* init buffers */ + if (rank != dest) { + for (i = 0; i < my_buf_num * OP_COUNT; i++) {orig_buf[i] = 1; result_buf[i] = 0;} + } + else { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + for (i = 0; i < WIN_BUF_NUM * OP_COUNT; i++) {target_buf[i] = 0;} + MPI_Win_unlock(rank, win); + } + + MPI_Win_lock_all(0, win); + if (rank != dest) { + for (i = 0; i < my_buf_num; i++) { + MPI_Get_accumulate(&(orig_buf[i*OP_COUNT]), 1, origin_dtp, + &(result_buf[i*OP_COUNT]), 1, origin_dtp, + dest, 0, OP_COUNT, MPI_INT, MPI_SUM, win); + MPI_Win_flush(dest, win); + } + } + MPI_Win_unlock_all(win); + + MPI_Barrier(MPI_COMM_WORLD); + + checkResults(k, &errors); + } + + MPI_Win_free(&win); + + if (rank == origin_am || rank == origin_shm) { + MPI_Free_mem(orig_buf); + MPI_Free_mem(result_buf); + } + + MPI_Type_free(&origin_dtp); + MPI_Type_free(&target_dtp); + + exit_test: + MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + + if (rank == 0 && all_errors == 0) + printf(" No Errors\n"); + + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/rma/testlist.in b/test/mpi/rma/testlist.in index 9a8a964..d7c8835 100644 --- a/test/mpi/rma/testlist.in +++ b/test/mpi/rma/testlist.in @@ -128,6 +128,9 @@ win_shared_zerobyte 4 mpiversion=3.0 win_shared_put_flush_get 4 mpiversion=3.0 get-struct 2 at_complete 2 +atomic_rmw_fop 3 +atomic_rmw_cas 3 +atomic_rmw_gacc 3 @mpix@ aint 2 strict=false ## This test is not strictly correct. This was meant to test out the http://git.mpich.org/mpich.git/commitdiff/bad898f9df13b5060cbf43ee4acdb3b7b… commit bad898f9df13b5060cbf43ee4acdb3b7b4c9a0f7 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Wed Feb 4 05:21:04 2015 -0800 Bug-fix: guarantee atomicity for FOP and GACC. FOP, CAS and GACC are atomic "read-modify-write" operations, which means when the target window is defined on a SHM region, we need inter-process lock to guarantee the atomicity of the entire "read+OP". The current implementation is correct for SHM-based RMA operations, but not correct for AM-based RMA operations: for SHM-based operations, it protects the entire "read+OP", but for AM-based operations, it only protects the "OP" part. This patch fixes this issue by protecting the memory copy to temporary buffer and computation together for AM-based operations. Fix ticket 2226 Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/src/ch3u_handle_recv_req.c b/src/mpid/ch3/src/ch3u_handle_recv_req.c index c2e90b6..79dc318 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_req.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_req.c @@ -255,6 +255,9 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, rreq->dev.user_count * type_size, mpi_errno, "GACC resp. buffer"); + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) { MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, rreq->dev.user_count * type_size); @@ -268,6 +271,16 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPID_Segment_free(seg); } + /* accumulate data from tmp_buf into user_buf */ + mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf, + rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + resp_req->dev.OnFinal = MPIDI_CH3_ReqHandler_GaccumSendComplete; resp_req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GaccumSendComplete; resp_req->dev.target_win_handle = rreq->dev.target_win_handle; @@ -322,17 +335,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete( MPIDI_VC_t *vc, MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_GET_ACCUM_RESP); - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - /* accumulate data from tmp_buf into user_buf */ - mpi_errno = do_accumulate_op(rreq->dev.final_user_buf, rreq->dev.real_user_buf, - rreq->dev.user_count, rreq->dev.datatype, rreq->dev.op); - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - /* free the temporary buffer */ MPIR_Type_get_true_extent_impl(rreq->dev.datatype, &true_lb, &true_extent); MPIU_Free((char *) rreq->dev.final_user_buf + true_lb); @@ -1023,6 +1025,10 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); sreq->dev.user_buf = (void *)MPIU_Malloc(get_accum_pkt->count * type_size); + /* Perform ACCUMULATE OP */ + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { MPIU_Memcpy(sreq->dev.user_buf, get_accum_pkt->addr, get_accum_pkt->count * type_size); @@ -1037,6 +1043,21 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPID_Segment_free(seg); } + if (lock_entry->data == NULL) { + /* All data fits in packet header */ + mpi_errno = do_accumulate_op(get_accum_pkt->data, get_accum_pkt->addr, + get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + else { + mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->addr, + get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + } + + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + /* here we increment the Active Target counter to guarantee the GET-like operation are completed when counter reaches zero. */ win_ptr->at_completion_counter++; @@ -1096,25 +1117,6 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Loc MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg"); } - /* Perform ACCUMULATE OP */ - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); - - if (lock_entry->data == NULL) { - /* All data fits in packet header */ - mpi_errno = do_accumulate_op(get_accum_pkt->data, get_accum_pkt->addr, - get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - else { - mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->addr, - get_accum_pkt->count, get_accum_pkt->datatype, get_accum_pkt->op); - if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); - } - - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); - fn_exit: return mpi_errno; fn_fail: @@ -1149,6 +1151,9 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; fop_resp_pkt->immed_len = fop_pkt->immed_len; + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + /* copy data to resp pkt header */ void *src = fop_pkt->addr, *dest = fop_resp_pkt->data; mpi_errno = immed_copy(src, dest, (size_t)fop_resp_pkt->immed_len); @@ -1158,13 +1163,12 @@ static inline int perform_fop_in_lock_queue(MPID_Win *win_ptr, MPIDI_RMA_Lock_en if (fop_pkt->op != MPI_NO_OP) { MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); int one = 1; - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); (*uop)(fop_pkt->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); } + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 76ac30d..a64e97c 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -1098,6 +1098,9 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK; fop_resp_pkt->immed_len = fop_pkt->immed_len; + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); + /* copy data to resp pkt header */ void *src = fop_pkt->addr, *dest = fop_resp_pkt->data; mpi_errno = immed_copy(src, dest, (size_t)fop_resp_pkt->immed_len); @@ -1107,13 +1110,12 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, if (fop_pkt->op != MPI_NO_OP) { MPI_User_function *uop = MPIR_OP_HDL_TO_FN(fop_pkt->op); int one = 1; - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr); (*uop)(fop_pkt->data, fop_pkt->addr, &one, &(fop_pkt->datatype)); - if (win_ptr->shm_allocated == TRUE) - MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); } + if (win_ptr->shm_allocated == TRUE) + MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr); + /* send back the original data */ MPIU_THREAD_CS_ENTER(CH3COMM,vc); mpi_errno = MPIDI_CH3_iStartMsg(vc, fop_resp_pkt, sizeof(*fop_resp_pkt), &resp_req); http://git.mpich.org/mpich.git/commitdiff/65bf0d776e8932e7acb26714ee5a8c01e… commit 65bf0d776e8932e7acb26714ee5a8c01ebf004ca Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Wed Feb 4 16:48:40 2015 -0800 Add tests to test cases when both SHM window and non-SHM window exist. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/test/mpi/rma/testlist.in b/test/mpi/rma/testlist.in index 32cd86b..9a8a964 100644 --- a/test/mpi/rma/testlist.in +++ b/test/mpi/rma/testlist.in @@ -71,6 +71,7 @@ contention_putget 4 put_base 2 put_bottom 2 win_flavors 4 mpiversion=3.0 +win_flavors 3 mpiversion=3.0 manyrma2 2 timeLimit=500 manyrma2_shm 2 timeLimit=500 manyrma3 2 @@ -81,6 +82,7 @@ win_shared_noncontig 4 mpiversion=3.0 win_shared_noncontig_put 4 mpiversion=3.0 win_zero 4 mpiversion=3.0 @largetest@win_large_shm 4 mpiversion=3.0 +@largetest@win_large_shm 3 mpiversion=3.0 win_dynamic_acc 4 mpiversion=3.0 get_acc_local 1 mpiversion=3.0 linked_list 4 mpiversion=3.0 diff --git a/test/mpi/rma/win_large_shm.c b/test/mpi/rma/win_large_shm.c index fe730d2..583eab2 100644 --- a/test/mpi/rma/win_large_shm.c +++ b/test/mpi/rma/win_large_shm.c @@ -17,62 +17,71 @@ int main(int argc, char **argv) { MPI_Win win; MPI_Info win_info; MPI_Comm shared_comm; + int i; int shm_win_size = 1024 * 1024 * 1024 * sizeof(char); /* 1GB */ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - MPI_Info_create(&win_info); - MPI_Info_set(win_info, (char*)"alloc_shm", (char*)"true"); + for (i = 0; i < 2; i++) { + if (i == 0) { + MPI_Info_create(&win_info); + MPI_Info_set(win_info, (char*)"alloc_shm", (char*)"true"); + } + else { + win_info = MPI_INFO_NULL; + } - MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, my_rank, MPI_INFO_NULL, &shared_comm); + MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, my_rank, MPI_INFO_NULL, &shared_comm); - MPI_Comm_rank(shared_comm, &shared_rank); + MPI_Comm_rank(shared_comm, &shared_rank); - /* every processes allocate 1GB window memory */ - MPI_Win_allocate(shm_win_size, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); + /* every processes allocate 1GB window memory */ + MPI_Win_allocate(shm_win_size, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); - MPI_Win_free(&win); + MPI_Win_free(&win); - MPI_Win_allocate_shared(shm_win_size, sizeof(char), win_info, shared_comm, &mybase, &win); + MPI_Win_allocate_shared(shm_win_size, sizeof(char), win_info, shared_comm, &mybase, &win); - MPI_Win_free(&win); + MPI_Win_free(&win); - /* some processes allocate 1GB and some processes allocate zero bytes */ - if (my_rank % 2 == 0) - MPI_Win_allocate(shm_win_size, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); - else - MPI_Win_allocate(0, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); + /* some processes allocate 1GB and some processes allocate zero bytes */ + if (my_rank % 2 == 0) + MPI_Win_allocate(shm_win_size, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); + else + MPI_Win_allocate(0, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); - MPI_Win_free(&win); + MPI_Win_free(&win); - if (shared_rank % 2 == 0) - MPI_Win_allocate_shared(shm_win_size, sizeof(char), win_info, shared_comm, &mybase, &win); - else - MPI_Win_allocate_shared(0, sizeof(char), win_info, shared_comm, &mybase, &win); + if (shared_rank % 2 == 0) + MPI_Win_allocate_shared(shm_win_size, sizeof(char), win_info, shared_comm, &mybase, &win); + else + MPI_Win_allocate_shared(0, sizeof(char), win_info, shared_comm, &mybase, &win); - MPI_Win_free(&win); + MPI_Win_free(&win); - /* some processes allocate 1GB and some processes allocate smaller bytes */ - if (my_rank % 2 == 0) - MPI_Win_allocate(shm_win_size, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); - else - MPI_Win_allocate(shm_win_size/2, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); + /* some processes allocate 1GB and some processes allocate smaller bytes */ + if (my_rank % 2 == 0) + MPI_Win_allocate(shm_win_size, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); + else + MPI_Win_allocate(shm_win_size/2, sizeof(char), win_info, MPI_COMM_WORLD, &mybase, &win); - MPI_Win_free(&win); + MPI_Win_free(&win); - /* some processes allocate 1GB and some processes allocate smaller bytes */ - if (shared_rank % 2 == 0) - MPI_Win_allocate_shared(shm_win_size, sizeof(char), win_info, shared_comm, &mybase, &win); - else - MPI_Win_allocate_shared(shm_win_size/2, sizeof(char), win_info, shared_comm, &mybase, &win); + /* some processes allocate 1GB and some processes allocate smaller bytes */ + if (shared_rank % 2 == 0) + MPI_Win_allocate_shared(shm_win_size, sizeof(char), win_info, shared_comm, &mybase, &win); + else + MPI_Win_allocate_shared(shm_win_size/2, sizeof(char), win_info, shared_comm, &mybase, &win); - MPI_Win_free(&win); + MPI_Win_free(&win); - MPI_Comm_free(&shared_comm); + MPI_Comm_free(&shared_comm); - MPI_Info_free(&win_info); + if (i == 0) + MPI_Info_free(&win_info); + } if (my_rank == 0) printf(" No Errors\n"); http://git.mpich.org/mpich.git/commitdiff/8c5cb1e67248e29a7a5e4523b5adcfa04… commit 8c5cb1e67248e29a7a5e4523b5adcfa044ab8a93 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Wed Feb 4 16:36:57 2015 -0800 Bug-fix: making processes with SHM and without SHM win work corrrectly. In commit 7d71278, if node_comm is NULL (only self process is on that node), we call allocate_no_shm() in CH3 to allocate window. If node_comm is not NULL (more than one process is on the same node), we call allocate_shm() in Nemesis to allocate SHM window. However, the exchanged information amount (in MPI_Allgather) is different in allocate_no_shm() and allocate_shm(), which leads to wrong execution when both SHM window and non-SHM window exist. This patch fixes this issue. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index 8348475..288213f 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -319,46 +319,19 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * /* get the sizes of the windows and window objectsof all processes. allocate temp. buffer for communication */ - MPIU_CHKLMEM_MALLOC(tmp_buf, MPI_Aint *, 3*comm_size*sizeof(MPI_Aint), mpi_errno, "tmp_buf"); + MPIU_CHKLMEM_MALLOC(node_sizes, MPI_Aint *, node_size*sizeof(MPI_Aint), mpi_errno, "node_sizes"); /* FIXME: This needs to be fixed for heterogeneous systems */ - tmp_buf[3*rank] = (MPI_Aint) size; - tmp_buf[3*rank+1] = (MPI_Aint) disp_unit; - tmp_buf[3*rank+2] = (MPI_Aint) (*win_ptr)->handle; + node_sizes[node_rank] = (MPI_Aint) size; mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, - tmp_buf, 3 * sizeof(MPI_Aint), MPI_BYTE, - (*win_ptr)->comm_ptr, &errflag); + node_sizes, sizeof(MPI_Aint), MPI_BYTE, + node_comm_ptr, &errflag); MPIR_T_PVAR_TIMER_END(RMA, rma_wincreate_allgather); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); - if ((*win_ptr)->create_flavor != MPI_WIN_FLAVOR_SHARED) { - MPIU_CHKLMEM_MALLOC(node_sizes, MPI_Aint *, node_size*sizeof(MPI_Aint), mpi_errno, "node_sizes"); - for (i = 0; i < node_size; i++) node_sizes[i] = 0; - } - else { - node_sizes = (*win_ptr)->sizes; - } - (*win_ptr)->shm_segment_len = 0; - k = 0; - for (i = 0; i < comm_size; ++i) { - (*win_ptr)->sizes[i] = tmp_buf[k++]; - (*win_ptr)->disp_units[i] = (int) tmp_buf[k++]; - (*win_ptr)->all_win_handles[i] = (MPI_Win) tmp_buf[k++]; - - if ((*win_ptr)->create_flavor != MPI_WIN_FLAVOR_SHARED) { - /* If create flavor is not MPI_WIN_FLAVOR_SHARED, all processes on this - window may not be on the same node. Because we only need the sizes of local - processes (in order), we copy their sizes to a seperate array and keep them - in order, fur purpose of future use of calculating shm_base_addrs. */ - if ((*win_ptr)->comm_ptr->intranode_table[i] >= 0) { - MPIU_Assert((*win_ptr)->comm_ptr->intranode_table[i] < node_size); - node_sizes[(*win_ptr)->comm_ptr->intranode_table[i]] = (*win_ptr)->sizes[i]; - } - } - } for (i = 0; i < node_size; i++) { if (noncontig) @@ -529,18 +502,29 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * (*win_ptr)->base = (*win_ptr)->shm_base_addrs[rank]; } + MPIU_CHKLMEM_MALLOC(tmp_buf, MPI_Aint *, 4*comm_size*sizeof(MPI_Aint), + mpi_errno, "tmp_buf"); + /* get the base addresses of the windows. Note we reuse tmp_buf from above since it's at least as large as we need it for this allgather. */ - tmp_buf[rank] = MPIU_PtrToAint((*win_ptr)->base); + tmp_buf[4*rank] = MPIU_PtrToAint((*win_ptr)->base); + tmp_buf[4*rank+1] = size; + tmp_buf[4*rank+2] = (MPI_Aint) disp_unit; + tmp_buf[4*rank+3] = (MPI_Aint) (*win_ptr)->handle; mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, - tmp_buf, 1, MPI_AINT, + tmp_buf, 4, MPI_AINT, (*win_ptr)->comm_ptr, &errflag); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); - for (i = 0; i < comm_size; ++i) - (*win_ptr)->base_addrs[i] = MPIU_AintToPtr(tmp_buf[i]); + k = 0; + for (i = 0; i < comm_size; ++i) { + (*win_ptr)->base_addrs[i] = MPIU_AintToPtr(tmp_buf[k++]); + (*win_ptr)->sizes[i] = tmp_buf[k++]; + (*win_ptr)->disp_units[i] = (int) tmp_buf[k++]; + (*win_ptr)->all_win_handles[i] = (MPI_Win) tmp_buf[k++]; + } *base_pp = (*win_ptr)->base; http://git.mpich.org/mpich.git/commitdiff/346050ea58448a53995b1581bb82b291f… commit 346050ea58448a53995b1581bb82b291f4a2d5f7 Author: Xin Zhao <xinzhao3(a)illinois.edu> Date: Wed Feb 4 16:29:40 2015 -0800 Delete unnecessary code in SHM allocate / free. We allocate / free SHM regions only when node_comm exists, which means there are more than one processes on the same node. When node_comm is NULL (only self process is on that node), we call default allocate / free functions in CH3. (Please refer to commit f02eed5b) Here we delete unnecessary code dealing with node_comm being NULL in SHM allocate / free functions. Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c index fa4d906..63c85f3 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c @@ -107,10 +107,7 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) that are on the same node as this process (node_comm). If node_comm == NULL, this process is the only one on this node, therefore we use comm_self as node comm. */ - if ((*win_ptr)->comm_ptr->node_comm != NULL) - node_comm_ptr = (*win_ptr)->comm_ptr->node_comm; - else - node_comm_ptr = MPIR_Process.comm_self; + node_comm_ptr = (*win_ptr)->comm_ptr->node_comm; MPIU_Assert(node_comm_ptr != NULL); if (node_comm_ptr->rank == 0) { diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index efd16cb..8348475 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -291,10 +291,7 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * that are on the same node as this process (node_comm). If node_comm == NULL, this process is the only one on this node, therefore we use comm_self as node comm. */ - if ((*win_ptr)->comm_ptr->node_comm != NULL) - node_comm_ptr = (*win_ptr)->comm_ptr->node_comm; - else - node_comm_ptr = MPIR_Process.comm_self; + node_comm_ptr = (*win_ptr)->comm_ptr->node_comm; MPIU_Assert(node_comm_ptr != NULL); node_size = node_comm_ptr->local_size; node_rank = node_comm_ptr->rank; ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c | 5 +- src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c | 59 ++---- src/mpid/ch3/src/ch3u_handle_recv_req.c | 72 ++++---- src/mpid/ch3/src/ch3u_rma_pkthandler.c | 10 +- test/mpi/rma/Makefile.am | 5 +- test/mpi/rma/atomic_rmw_cas.c | 129 ++++++++++++ test/mpi/rma/atomic_rmw_fop.c | 131 ++++++++++++ test/mpi/rma/atomic_rmw_gacc.c | 240 +++++++++++++++++++++++ test/mpi/rma/testlist.in | 5 + test/mpi/rma/win_large_shm.c | 77 ++++---- 10 files changed, 617 insertions(+), 116 deletions(-) create mode 100644 test/mpi/rma/atomic_rmw_cas.c create mode 100644 test/mpi/rma/atomic_rmw_fop.c create mode 100644 test/mpi/rma/atomic_rmw_gacc.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-136-g7dfe284
by noreply＠mpich.org 05 Feb '15

05 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 7dfe284051cacc8ef79f15e0c01c81b28f9fe78e (commit) from b0dcf8cde20879b39ae1edfe1deb544f70c06147 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/7dfe284051cacc8ef79f15e0c01c81b28… commit 7dfe284051cacc8ef79f15e0c01c81b28f9fe78e Author: Rajeev Thakur <thakur(a)mcs.anl.gov> Date: Tue Feb 3 19:54:50 2015 -0600 Fix MPI_Info_get to pass valuelen+1 to MPIU_Strncpy and check return code of MPIU_Strncpy. Added test program. Closes #2225 Signed-off-by: William Gropp <wgropp(a)illinois.edu> diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h index 4204466..cc1a457 100644 --- a/src/include/mpiimpl.h +++ b/src/include/mpiimpl.h @@ -4418,7 +4418,7 @@ int MPIR_Cart_map_impl(const MPID_Comm *comm_ptr, int ndims, const int dims[], const int periodic[], int *newrank); int MPIR_Close_port_impl(const char *port_name); int MPIR_Open_port_impl(MPID_Info *info_ptr, char *port_name); -void MPIR_Info_get_impl(MPID_Info *info_ptr, const char *key, int valuelen, char *value, int *flag); +int MPIR_Info_get_impl(MPID_Info *info_ptr, const char *key, int valuelen, char *value, int *flag); void MPIR_Info_get_nkeys_impl(MPID_Info *info_ptr, int *nkeys); int MPIR_Info_get_nthkey_impl(MPID_Info *info, int n, char *key); void MPIR_Info_get_valuelen_impl(MPID_Info *info_ptr, const char *key, int *valuelen, int *flag); diff --git a/src/mpi/info/info_get.c b/src/mpi/info/info_get.c index 690c38a..cae3f46 100644 --- a/src/mpi/info/info_get.c +++ b/src/mpi/info/info_get.c @@ -29,28 +29,35 @@ int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, int #define FUNCNAME MPIR_Info_get_impl #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -void MPIR_Info_get_impl(MPID_Info *info_ptr, const char *key, int valuelen, char *value, int *flag) +int MPIR_Info_get_impl(MPID_Info *info_ptr, const char *key, int valuelen, char *value, int *flag) { MPID_Info *curr_ptr; + int err=0, mpi_errno=0; + curr_ptr = info_ptr->next; *flag = 0; while (curr_ptr) { if (!strncmp(curr_ptr->key, key, MPI_MAX_INFO_KEY)) { - MPIU_Strncpy(value, curr_ptr->value, valuelen); - /* The following is problematic - if the user passes the - declared length, then this will access memory one - passed that point */ - /* FIXME: The real fix is to change MPIU_Strncpy to - set the null at the end (always!) and return an error - if it had to truncate the result. */ - /* value[valuelen] = '\0'; */ + err = MPIU_Strncpy(value, curr_ptr->value, valuelen+1); + /* +1 because the MPI Standard says "In C, valuelen + * (passed to MPI_Info_get) should be one less than the + * amount of allocated space to allow for the null + * terminator*/ *flag = 1; break; } curr_ptr = curr_ptr->next; } - return; + + /* --BEGIN ERROR HANDLING-- */ + if (err != 0) + { + mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_INFO_VALUE, "**infovallong", NULL); + } + /* --END ERROR HANDLING-- */ + + return mpi_errno; } #endif @@ -61,7 +68,7 @@ void MPIR_Info_get_impl(MPID_Info *info_ptr, const char *key, int valuelen, char Input Parameters: + info - info object (handle) . key - key (string) -- valuelen - length of value argument (integer) +- valuelen - length of value argument, not including null terminator (integer) Output Parameters: + value - value (string) @@ -139,8 +146,9 @@ int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, # endif /* HAVE_ERROR_CHECKING */ /* ... body of routine ... */ - MPIR_Info_get_impl(info_ptr, key, valuelen, value, flag); + mpi_errno = MPIR_Info_get_impl(info_ptr, key, valuelen, value, flag); /* ... end of body of routine ... */ + if (mpi_errno) goto fn_fail; #ifdef HAVE_ERROR_CHECKING fn_exit: diff --git a/test/mpi/info/Makefile.am b/test/mpi/info/Makefile.am index 956fa90..1bc23a8 100644 --- a/test/mpi/info/Makefile.am +++ b/test/mpi/info/Makefile.am @@ -20,4 +20,5 @@ noinst_PROGRAMS = \ infomany \ infomany2 \ infotest \ + infoget \ infoenv diff --git a/test/mpi/info/infoget.c b/test/mpi/info/infoget.c new file mode 100644 index 0000000..cdc0f1d --- /dev/null +++ b/test/mpi/info/infoget.c @@ -0,0 +1,42 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +/* Test code provided by Hajime Fujita. See Trac ticket #2225. */ + +#include "mpi.h" +#include <stdio.h> +#include "mpitest.h" +#include <string.h> + +int main(int argc, char *argv[]) +{ + MPI_Info info; + const char *key = "key", *val = "val"; + char buff[3 + 1]; /* strlen("val") + 1 */ + int flag, errs=0; + + MTest_Init(&argc, &argv); + + MPI_Info_create(&info); + MPI_Info_set(info, key, val); + MPI_Info_get(info, key, sizeof(buff)-1, buff, &flag); + if (flag) { + if (strncmp(buff, val, sizeof(buff)-1) != 0) { + errs++; + printf("returned value is %s, should be %s\n", buff, val); + } + } + else { + errs++; + printf("key not found\n"); + } + MPI_Info_free(&info); + + MTest_Finalize(errs); + MPI_Finalize(); + + return 0; +} diff --git a/test/mpi/info/testlist b/test/mpi/info/testlist index 724a6a2..b174c00 100644 --- a/test/mpi/info/testlist +++ b/test/mpi/info/testlist @@ -5,4 +5,5 @@ infoorder 1 infomany 1 infomany2 1 infotest 1 +infoget 1 infoenv 1 mpiversion=3.0 ----------------------------------------------------------------------- Summary of changes: src/include/mpiimpl.h | 2 +- src/mpi/info/info_get.c | 32 ++++++++++++++++++++------------ test/mpi/info/Makefile.am | 1 + test/mpi/info/infoget.c | 42 ++++++++++++++++++++++++++++++++++++++++++ test/mpi/info/testlist | 1 + 5 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 test/mpi/info/infoget.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-135-gb0dcf8c
by noreply＠mpich.org 04 Feb '15

04 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via b0dcf8cde20879b39ae1edfe1deb544f70c06147 (commit) from 1260f270d6f51971e52da8e83c535bd7d2901c6e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/b0dcf8cde20879b39ae1edfe1deb544f7… commit b0dcf8cde20879b39ae1edfe1deb544f70c06147 Author: Wesley Bland <wbland(a)anl.gov> Date: Wed Feb 4 13:25:34 2015 -0600 Add weak symbols for MPIX functions Some of the MPIX functions did not have weak symbols set up correctly which causes problems on some compilers (Pathscale). This patch adds the correct attribute for all of them that were missing. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpi/comm/comm_agree.c b/src/mpi/comm/comm_agree.c index e0b47d7..e3594f2 100644 --- a/src/mpi/comm/comm_agree.c +++ b/src/mpi/comm/comm_agree.c @@ -15,6 +15,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Comm_agree MPIX_Comm_agree #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Comm_agree as PMPIX_Comm_agree +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Comm_agree(MPI_Comm comm, int *flag) __attribute__((weak,alias("PMPIX_Comm_agree"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mpi/comm/comm_failure_ack.c b/src/mpi/comm/comm_failure_ack.c index 4337eb3..14c929e 100644 --- a/src/mpi/comm/comm_failure_ack.c +++ b/src/mpi/comm/comm_failure_ack.c @@ -15,6 +15,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Comm_failure_ack MPIX_Comm_failure_ack #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Comm_failure_ack as PMPIX_Comm_failure_ack +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Comm_failure_ack( MPI_Comm comm ) __attribute__((weak,alias("PMPIX_Comm_failure_ack"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mpi/comm/comm_failure_get_acked.c b/src/mpi/comm/comm_failure_get_acked.c index aac0c95..2289a89 100644 --- a/src/mpi/comm/comm_failure_get_acked.c +++ b/src/mpi/comm/comm_failure_get_acked.c @@ -15,6 +15,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Comm_failure_get_acked MPIX_Comm_failure_get_acked #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Comm_failure_get_acked as PMPIX_Comm_failure_get_acked +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Comm_failure_get_acked( MPI_Comm comm, MPI_Group *failedgrp ) __attribute__((weak,alias("PMPIX_Comm_failure_get_acked"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mpi/comm/comm_revoke.c b/src/mpi/comm/comm_revoke.c index 7a5154f..d134e67 100644 --- a/src/mpi/comm/comm_revoke.c +++ b/src/mpi/comm/comm_revoke.c @@ -17,6 +17,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Comm_revoke MPIX_Comm_revoke #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Comm_revoke as PMPIX_Comm_revoke +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Comm_revoke(MPI_Comm comm) __attribute__((weak,alias("PMPIX_Comm_revoke"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mpi/comm/comm_shrink.c b/src/mpi/comm/comm_shrink.c index 0466db5..56536d7 100644 --- a/src/mpi/comm/comm_shrink.c +++ b/src/mpi/comm/comm_shrink.c @@ -29,6 +29,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Comm_shrink MPIX_Comm_shrink #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Comm_shrink as PMPIX_Comm_shrink +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm) __attribute__((weak,alias("PMPIX_Comm_shrink"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mpi/pt2pt/greq_start.c b/src/mpi/pt2pt/greq_start.c index 3bd9a77..5ec7075 100644 --- a/src/mpi/pt2pt/greq_start.c +++ b/src/mpi/pt2pt/greq_start.c @@ -230,6 +230,13 @@ int MPI_Grequest_start( MPI_Grequest_query_function *query_fn, #pragma _HP_SECONDARY_DEF PMPIX_Grequest_class_create MPIX_Grequest_class_create #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Grequest_class_create as PMPIX_Grequest_class_create +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Grequest_class_create(MPI_Grequest_query_function *query_fn, + MPI_Grequest_free_function *free_fn, + MPI_Grequest_cancel_function *cancel_fn, + MPIX_Grequest_poll_function *poll_fn, + MPIX_Grequest_wait_function *wait_fn, + MPIX_Grequest_class *greq_class) __attribute__((weak,alias("MPIX_Grequest_class_create"))); #endif /* -- End Profiling Symbol Block */ @@ -320,6 +327,8 @@ fn_fail: #pragma _HP_SECONDARY_DEF PMPI_Grequest_class_allocate MPIX_Grequest_class_allocate #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Grequest_class_allocate as PMPIX_Grequest_class_allocate +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Grequest_class_allocate(MPIX_Grequest_class greq_class, void *extra_state, MPI_Request *request) __attribute__((weak,alias("MPIX_Grequest_class_allocate"))); #endif /* -- End Profiling Symbol Block */ @@ -363,6 +372,14 @@ int MPIX_Grequest_class_allocate(MPIX_Grequest_class greq_class, #pragma _HP_SECONDARY_DEF PMPI_Grequest_start MPIX_Grequest_start #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Grequest_start as PMPIX_Grequest_start +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Grequest_start( MPI_Grequest_query_function *query_fn, + MPI_Grequest_free_function *free_fn, + MPI_Grequest_cancel_function *cancel_fn, + MPIX_Grequest_poll_function *poll_fn, + MPIX_Grequest_wait_function *wait_fn, + void *extra_state, + MPI_Request *request ) __attribute__((weak,alias("MPIX_Grequest_start"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mutex/mutex_create.c b/src/mutex/mutex_create.c index 5bae747..eec39c1 100644 --- a/src/mutex/mutex_create.c +++ b/src/mutex/mutex_create.c @@ -22,6 +22,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Mutex_create MPIX_Mutex_create #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Mutex_create as PMPIX_Mutex_create +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Mutex_create(int my_count, MPI_Comm comm, MPIX_Mutex * hdl_out) __attribute__((weak,alias("MPIX_Mutex_create"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mutex/mutex_free.c b/src/mutex/mutex_free.c index 767c00b..79715ca 100644 --- a/src/mutex/mutex_free.c +++ b/src/mutex/mutex_free.c @@ -19,6 +19,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Mutex_free MPIX_Mutex_free #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Mutex_free as PMPIX_Mutex_free +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Mutex_free(MPIX_Mutex * hdl_ptr) __attribute__((weak,alias("MPIX_Mutex_free"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mutex/mutex_lock.c b/src/mutex/mutex_lock.c index d6f9836..257e80f 100644 --- a/src/mutex/mutex_lock.c +++ b/src/mutex/mutex_lock.c @@ -20,6 +20,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Mutex_lock MPIX_Mutex_lock #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Mutex_lock as PMPIX_Mutex_lock +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Mutex_lock(MPIX_Mutex hdl, int mutex, int proc) __attribute__((weak,alias("MPIX_Mutex_lock"))); #endif /* -- End Profiling Symbol Block */ diff --git a/src/mutex/mutex_unlock.c b/src/mutex/mutex_unlock.c index 5dd67d9..722f8b9 100644 --- a/src/mutex/mutex_unlock.c +++ b/src/mutex/mutex_unlock.c @@ -20,6 +20,8 @@ #pragma _HP_SECONDARY_DEF PMPIX_Mutex_unlock MPIX_Mutex_unlock #elif defined(HAVE_PRAGMA_CRI_DUP) #pragma _CRI duplicate MPIX_Mutex_unlock as PMPIX_Mutex_unlock +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPIX_Mutex_unlock(MPIX_Mutex hdl, int mutex, int proc) __attribute__((weak,alias("MPIX_Mutex_unlock"))); #endif /* -- End Profiling Symbol Block */ ----------------------------------------------------------------------- Summary of changes: src/mpi/comm/comm_agree.c | 2 ++ src/mpi/comm/comm_failure_ack.c | 2 ++ src/mpi/comm/comm_failure_get_acked.c | 2 ++ src/mpi/comm/comm_revoke.c | 2 ++ src/mpi/comm/comm_shrink.c | 2 ++ src/mpi/pt2pt/greq_start.c | 17 +++++++++++++++++ src/mutex/mutex_create.c | 2 ++ src/mutex/mutex_free.c | 2 ++ src/mutex/mutex_lock.c | 2 ++ src/mutex/mutex_unlock.c | 2 ++ 10 files changed, 35 insertions(+), 0 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-134-g1260f27
by noreply＠mpich.org 04 Feb '15

04 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 1260f270d6f51971e52da8e83c535bd7d2901c6e (commit) via d40ff05689444ac8fe4ced791aa7ee4847e58f28 (commit) from ebbaad2be4676c23586685950618aff6229b1ad2 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/1260f270d6f51971e52da8e83c535bd7d… commit 1260f270d6f51971e52da8e83c535bd7d2901c6e Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Mon Feb 2 15:57:15 2015 -0600 portals4: simplify send callback Merges the existing send callbacks into a single function. Uses the completion counter to track remaining operations and complete the request once finished. Signed-off-by: Antonio Pena Monferrer <apenya(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c index d3f4d91..4ee5de1 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c @@ -13,7 +13,7 @@ #define FCNAME MPIU_QUOTE(FUNCNAME) static void big_meappend(void *buf, ptl_size_t left_to_send, MPIDI_VC_t *vc, ptl_match_bits_t match_bits, MPID_Request *sreq) { - int i, ret; + int i, ret, was_incomplete; MPID_nem_ptl_vc_area *vc_ptl; ptl_me_t me; @@ -45,77 +45,54 @@ static void big_meappend(void *buf, ptl_size_t left_to_send, MPIDI_VC_t *vc, ptl &REQ_PTL(sreq)->get_me_p[i]); DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq); MPIU_Assert(ret == 0); + /* increment the cc for each get operation */ + MPIDI_CH3U_Request_increment_cc(sreq, &was_incomplete); + MPIU_Assert(was_incomplete); + REQ_PTL(sreq)->num_gets++; /* account for what has been sent */ me.start = (char *)me.start + me.length; left_to_send -= me.length; - REQ_PTL(sreq)->num_gets++; } } #undef FUNCNAME -#define FUNCNAME handler_send_complete +#define FUNCNAME handler_send #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -static int handler_send_complete(const ptl_event_t *e) +static int handler_send(const ptl_event_t *e) { int mpi_errno = MPI_SUCCESS; MPID_Request *const sreq = e->user_ptr; - int ret; - int i; - MPIDI_STATE_DECL(MPID_STATE_HANDLER_SEND_COMPLETE); + int i, ret, incomplete; + + MPIDI_STATE_DECL(MPID_STATE_HANDLER_SEND); - MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_SEND_COMPLETE); + MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_SEND); MPIU_Assert(e->type == PTL_EVENT_SEND || e->type == PTL_EVENT_GET); - if (REQ_PTL(sreq)->md != PTL_INVALID_HANDLE) { - ret = PtlMDRelease(REQ_PTL(sreq)->md); - MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdrelease", "**ptlmdrelease %s", MPID_nem_ptl_strerror(ret)); - } + /* if we are done, release all resources and complete the request */ + if (sreq->cc == 1) { + if (REQ_PTL(sreq)->md != PTL_INVALID_HANDLE) { + ret = PtlMDRelease(REQ_PTL(sreq)->md); + MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdrelease", "**ptlmdrelease %s", MPID_nem_ptl_strerror(ret)); + } - for (i = 0; i < MPID_NEM_PTL_NUM_CHUNK_BUFFERS; ++i) - if (REQ_PTL(sreq)->chunk_buffer[i]) - MPIU_Free(REQ_PTL(sreq)->chunk_buffer[i]); + for (i = 0; i < MPID_NEM_PTL_NUM_CHUNK_BUFFERS; ++i) + if (REQ_PTL(sreq)->chunk_buffer[i]) + MPIU_Free(REQ_PTL(sreq)->chunk_buffer[i]); - if (REQ_PTL(sreq)->get_me_p) - MPIU_Free(REQ_PTL(sreq)->get_me_p); + if (REQ_PTL(sreq)->get_me_p) + MPIU_Free(REQ_PTL(sreq)->get_me_p); - MPIDI_CH3U_Request_complete(sreq); - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_SEND_COMPLETE); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME handler_large -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static int handler_large(const ptl_event_t *e) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Request *const sreq = e->user_ptr; - MPIDI_STATE_DECL(MPID_STATE_HANDLER_LARGE); - - MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_LARGE); - - MPIU_Assert(e->type == PTL_EVENT_SEND || e->type == PTL_EVENT_GET); - - if (e->type == PTL_EVENT_SEND) { - REQ_PTL(sreq)->put_done = 1; - } else if (e->type == PTL_EVENT_GET) { - /* decrement the remaining get operations */ - REQ_PTL(sreq)->num_gets--; + MPIDI_CH3U_Request_complete(sreq); + } else { + MPIDI_CH3U_Request_decrement_cc(sreq, &incomplete); } - if (REQ_PTL(sreq)->num_gets == 0 && REQ_PTL(sreq)->put_done) - mpi_errno = handler_send_complete(e); - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_LARGE); + MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_SEND); return mpi_errno; fn_fail: goto fn_exit; @@ -163,7 +140,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * /* Small message. Send all data eagerly */ if (dt_contig) { MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Small contig message"); - REQ_PTL(sreq)->event_handler = handler_send_complete; + REQ_PTL(sreq)->event_handler = handler_send; MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "&REQ_PTL(sreq)->event_handler = %p", &(REQ_PTL(sreq)->event_handler)); ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)((char *)buf + dt_true_lb), data_sz, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq, @@ -201,7 +178,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(sreq)->md); MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret)); - REQ_PTL(sreq)->event_handler = handler_send_complete; + REQ_PTL(sreq)->event_handler = handler_send; ret = MPID_nem_ptl_rptl_put(REQ_PTL(sreq)->md, 0, data_sz, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq, NPTL_HEADER(ssend_flag, data_sz)); @@ -218,7 +195,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * last = data_sz; MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, REQ_PTL(sreq)->chunk_buffer[0]); MPIU_Assert(last == sreq->dev.segment_size); - REQ_PTL(sreq)->event_handler = handler_send_complete; + REQ_PTL(sreq)->event_handler = handler_send; ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)REQ_PTL(sreq)->chunk_buffer[0], data_sz, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq, NPTL_HEADER(ssend_flag, data_sz)); @@ -235,7 +212,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), sreq); REQ_PTL(sreq)->large = TRUE; - REQ_PTL(sreq)->event_handler = handler_large; + REQ_PTL(sreq)->event_handler = handler_send; ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)((char *)buf + dt_true_lb), PTL_LARGE_THRESHOLD, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq, NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz)); @@ -273,6 +250,8 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * if (last == sreq->dev.segment_size && last <= MPIDI_nem_ptl_ni_limits.max_msg_size + PTL_LARGE_THRESHOLD) { /* Entire message fit in one IOV */ + int was_incomplete; + MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, " rest of message fits in one IOV"); /* Create ME for remaining data */ me.start = &sreq->dev.iov[initial_iov_count]; @@ -288,11 +267,14 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * MPIU_CHKPMEM_MALLOC(REQ_PTL(sreq)->get_me_p, ptl_handle_me_t *, sizeof(ptl_handle_me_t), mpi_errno, "get_me_p"); - REQ_PTL(sreq)->num_gets = 1; ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_get_pt, &me, PTL_PRIORITY_LIST, sreq, &REQ_PTL(sreq)->get_me_p[0]); MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret)); DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq); + /* increment the cc for the get operation */ + MPIDI_CH3U_Request_increment_cc(sreq, &was_incomplete); + MPIU_Assert(was_incomplete); + REQ_PTL(sreq)->num_gets = 1; /* Create MD for first chunk */ md.start = sreq->dev.iov; @@ -305,7 +287,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * REQ_PTL(sreq)->large = TRUE; - REQ_PTL(sreq)->event_handler = handler_large; + REQ_PTL(sreq)->event_handler = handler_send; ret = MPID_nem_ptl_rptl_put(REQ_PTL(sreq)->md, 0, PTL_LARGE_THRESHOLD, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq, NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz)); @@ -329,7 +311,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), sreq); REQ_PTL(sreq)->large = TRUE; - REQ_PTL(sreq)->event_handler = handler_large; + REQ_PTL(sreq)->event_handler = handler_send; ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)REQ_PTL(sreq)->chunk_buffer[0], PTL_LARGE_THRESHOLD, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq, NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz)); http://git.mpich.org/mpich.git/commitdiff/d40ff05689444ac8fe4ced791aa7ee484… commit d40ff05689444ac8fe4ced791aa7ee4847e58f28 Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Mon Feb 2 10:32:03 2015 -0600 portals4: remove dead code Signed-off-by: Antonio Pena Monferrer <apenya(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c index e49546f..d3f4d91 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c @@ -121,99 +121,6 @@ static int handler_large(const ptl_event_t *e) goto fn_exit; } -#if 0 - -#undef FUNCNAME -#define FUNCNAME handler_pack_chunk -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static int handler_pack_chunk(const ptl_event_t *e) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Request *const sreq = e->user_ptr; - MPIDI_STATE_DECL(MPID_STATE_HANDLER_PACK_CHUNK); - - MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_PACK_CHUNK); - - MPIU_Assert(e->type == PTL_EVENT_GET || e->type == PTL_EVENT_PUT); - - if (e->type == PTL_EVENT_PUT) { - mpi_errno = handler_send_complete(e); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - goto fn_exit; - } - - /* pack next chunk */ - MPI_nem_ptl_pack_byte(sreq->dev.segment_ptr, sreq->dev.segment_first, sreq->dev.segment_first + PTL_LARGE_THRESHOLD, - REQ_PTL(sreq_)->chunk_buffer[1], &REQ_PTL(sreq)->overflow[1]); - sreq->dev.segment_first += PTL_LARGE_THRESHOLD; - - /* notify receiver */ - ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, 0, 0, PTL_ACK_REQ, vc_ptl->id, - vc_ptl->pt, ?????, 0, sreq, - NPTL_HEADER(?????, MPIDI_Process.my_pg_rank, me.match_bits)); - - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_PACK_CHUNK); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#undef FUNCNAME -#define FUNCNAME handler_multi_put -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static int handler_multi_put(const ptl_event_t *e) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Request *const sreq = e->user_ptr; - MPIDI_STATE_DECL(MPID_STATE_HANDLER_MULTI_PUT); - - MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_MULTI_PUT); - - - - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_MULTI_PUT); - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME -#define FUNCNAME handler_large_multi -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -static int handler_large_multi(const ptl_event_t *e) -{ - int mpi_errno = MPI_SUCCESS; - MPID_Request *const sreq = e->user_ptr; - MPIDI_STATE_DECL(MPID_STATE_HANDLER_LARGE_MULTI); - - MPIU_Assert(e->type == PTL_EVENT_ACK); - - MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_LARGE_MULTI); - if (e->mlength < PTL_LARGE_THRESHOLD) { - /* truncated message */ - mpi_errno = handler_send_complete(e); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } else { - REQ_PTL(sreq)->event_handler = handler_pack_chunk; - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_LARGE_MULTI); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -#endif - - /* Send message for either isend or issend */ #undef FUNCNAME #define FUNCNAME send_msg ----------------------------------------------------------------------- Summary of changes: .../channels/nemesis/netmod/portals4/ptl_send.c | 185 ++++---------------- 1 files changed, 37 insertions(+), 148 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-132-gebbaad2
by noreply＠mpich.org 03 Feb '15

03 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via ebbaad2be4676c23586685950618aff6229b1ad2 (commit) from 9a9ffb76c216ae3051e8f02163ba30abf55eeaf0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/ebbaad2be4676c23586685950618aff62… commit ebbaad2be4676c23586685950618aff6229b1ad2 Author: Rajeev Thakur <thakur(a)mcs.anl.gov> Date: Tue Jan 27 14:51:42 2015 -0600 Changes the sticky lb/ub fields in resized types to 0, since the lb/ub set by type_create_resized are not sticky. Changes darray and subarray types to use type_create_resized instead of type_struct with explicit lb/ub, because explicit MPI_LB/MPI_UB have been removed from MPI in MPI-3 and they also cause other problems because they were defined to be sticky in MPI-1. Fixes type_create_struct, which was incorrectly setting lb and ub to true_lb and true_ub in the non-sticky case. Closes #2218 Closes #2220 Closes #2224 Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/datatype/type_create_darray.c b/src/mpi/datatype/type_create_darray.c index 0047f38..5c85f47 100644 --- a/src/mpi/datatype/type_create_darray.c +++ b/src/mpi/datatype/type_create_darray.c @@ -197,7 +197,7 @@ PMPI_LOCAL int MPIR_Type_cyclic(const int *array_of_gsizes, int mpi_errno,blksize, i, blklens[3], st_index, end_index, local_size, rem, count; MPI_Aint stride, disps[3]; - MPI_Datatype type_tmp, types[3]; + MPI_Datatype type_tmp, type_indexed, types[3]; if (darg == MPI_DISTRIBUTE_DFLT_DARG) blksize = 1; else blksize = darg; @@ -281,18 +281,30 @@ PMPI_LOCAL int MPIR_Type_cyclic(const int *array_of_gsizes, if (((order == MPI_ORDER_FORTRAN) && (dim == 0)) || ((order == MPI_ORDER_C) && (dim == ndims-1))) { - types[0] = MPI_LB; disps[0] = 0; - types[1] = *type_new; disps[1] = (MPI_Aint) rank * (MPI_Aint) blksize * orig_extent; - types[2] = MPI_UB; disps[2] = orig_extent * (MPI_Aint)(array_of_gsizes[dim]); - blklens[0] = blklens[1] = blklens[2] = 1; - mpi_errno = MPID_Type_struct(3, - blklens, - disps, - types, - &type_tmp); + +/* Instead of using MPI_LB/MPI_UB, which have been removed from MPI in MPI-3, + use MPI_Type_create_resized. Use hindexed_block to set the starting displacement + of the datatype (disps[1]) and type_create_resized to set lb to 0 (disps[0]) + and extent to disps[2], which makes ub = disps[2]. + */ + mpi_errno = MPID_Type_blockindexed(1, 1, &disps[1], + 1, /* 1 means disp is in bytes */ + *type_new, &type_indexed); + + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) + { + mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**fail", 0); + return mpi_errno; + } + /* --END ERROR HANDLING-- */ + + mpi_errno = MPID_Type_create_resized(type_indexed, 0, disps[2], &type_tmp); + + MPIR_Type_free_impl(&type_indexed); MPIR_Type_free_impl(type_new); *type_new = type_tmp; @@ -364,9 +376,9 @@ int MPI_Type_create_darray(int size, int mpi_errno = MPI_SUCCESS, i; MPI_Datatype new_handle; - int procs, tmp_rank, tmp_size, blklens[3], *coords; + int procs, tmp_rank, tmp_size, *coords; MPI_Aint *st_offsets, orig_extent, disps[3]; - MPI_Datatype type_old, type_new = MPI_DATATYPE_NULL, types[3]; + MPI_Datatype type_old, type_new = MPI_DATATYPE_NULL, tmp_type; # ifdef HAVE_ERROR_CHECKING MPI_Aint size_with_aint; @@ -666,20 +678,27 @@ int MPI_Type_create_darray(int size, for (i=0; i<ndims; i++) disps[2] *= (MPI_Aint)(array_of_gsizes[i]); disps[0] = 0; - blklens[0] = blklens[1] = blklens[2] = 1; - types[0] = MPI_LB; - types[1] = type_new; - types[2] = MPI_UB; - - mpi_errno = MPID_Type_struct(3, - blklens, - disps, - types, - &new_handle); + +/* Instead of using MPI_LB/MPI_UB, which have been removed from MPI in MPI-3, + use MPI_Type_create_resized. Use hindexed_block to set the starting displacement + of the datatype (disps[1]) and type_create_resized to set lb to 0 (disps[0]) + and extent to disps[2], which makes ub = disps[2]. + */ + mpi_errno = MPID_Type_blockindexed(1, 1, &disps[1], + 1, /* 1 means disp is in bytes */ + type_new, &tmp_type); + + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) goto fn_fail; + /* --END ERROR HANDLING-- */ + + mpi_errno = MPID_Type_create_resized(tmp_type, 0, disps[2], &new_handle); + /* --BEGIN ERROR HANDLING-- */ if (mpi_errno != MPI_SUCCESS) goto fn_fail; /* --END ERROR HANDLING-- */ + MPIR_Type_free_impl(&tmp_type); MPIR_Type_free_impl(&type_new); /* at this point we have the new type, and we've cleaned up any diff --git a/src/mpi/datatype/type_create_subarray.c b/src/mpi/datatype/type_create_subarray.c index 0089f37..906f865 100644 --- a/src/mpi/datatype/type_create_subarray.c +++ b/src/mpi/datatype/type_create_subarray.c @@ -72,8 +72,7 @@ int MPI_Type_create_subarray(int ndims, /* these variables are from the original version in ROMIO */ MPI_Aint size, extent, disps[3]; - int blklens[3]; - MPI_Datatype tmp1, tmp2, types[3]; + MPI_Datatype tmp1, tmp2; # ifdef HAVE_ERROR_CHECKING MPI_Aint size_with_aint; @@ -278,29 +277,23 @@ int MPI_Type_create_subarray(int ndims, for (i=0; i<ndims; i++) disps[2] *= (MPI_Aint)(array_of_sizes[i]); disps[0] = 0; - blklens[0] = blklens[1] = blklens[2] = 1; - types[0] = MPI_LB; - types[1] = tmp1; - types[2] = MPI_UB; - - /* TODO: - * if we were to do all this as an mpid function, we could just - * directly adjust the LB and UB in the MPID_Datatype structure - * instead of jumping through this hoop. - * - * i suppose we could do the same thing here... - * - * another alternative would be to use MPID_Type_create_resized() - * instead of building the struct. that would also be cleaner. - */ - mpi_errno = MPID_Type_struct(3, - blklens, - disps, - types, - &new_handle); + +/* Instead of using MPI_LB/MPI_UB, which have been removed from MPI in MPI-3, + use MPI_Type_create_resized. Use hindexed_block to set the starting displacement + of the datatype (disps[1]) and type_create_resized to set lb to 0 (disps[0]) + and extent to disps[2], which makes ub = disps[2]. + */ + + mpi_errno = MPID_Type_blockindexed(1, 1, &disps[1], + 1, /* 1 means disp is in bytes */ + tmp1, &tmp2); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + + mpi_errno = MPID_Type_create_resized(tmp2, 0, disps[2], &new_handle); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIR_Type_free_impl(&tmp1); + MPIR_Type_free_impl(&tmp2); /* at this point we have the new type, and we've cleaned up any * intermediate types created in the process. we just need to save diff --git a/src/mpid/common/datatype/mpid_type_create_resized.c b/src/mpid/common/datatype/mpid_type_create_resized.c index 07d753d..75b535f 100644 --- a/src/mpid/common/datatype/mpid_type_create_resized.c +++ b/src/mpid/common/datatype/mpid_type_create_resized.c @@ -47,8 +47,8 @@ int MPID_Type_create_resized(MPI_Datatype oldtype, int oldsize = MPID_Datatype_get_basic_size(oldtype); new_dtp->size = oldsize; - new_dtp->has_sticky_ub = 1; - new_dtp->has_sticky_lb = 1; + new_dtp->has_sticky_ub = 0; + new_dtp->has_sticky_lb = 0; new_dtp->dataloop_depth = 1; new_dtp->true_lb = 0; new_dtp->lb = lb; @@ -70,8 +70,8 @@ int MPID_Type_create_resized(MPI_Datatype oldtype, MPID_Datatype_get_ptr(oldtype, old_dtp); new_dtp->size = old_dtp->size; - new_dtp->has_sticky_ub = 1; - new_dtp->has_sticky_lb = 1; + new_dtp->has_sticky_ub = 0; + new_dtp->has_sticky_lb = 0; new_dtp->dataloop_depth = old_dtp->dataloop_depth; new_dtp->true_lb = old_dtp->true_lb; new_dtp->lb = lb; diff --git a/src/mpid/common/datatype/mpid_type_struct.c b/src/mpid/common/datatype/mpid_type_struct.c index 148958e..72e23fd 100644 --- a/src/mpid/common/datatype/mpid_type_struct.c +++ b/src/mpid/common/datatype/mpid_type_struct.c @@ -151,12 +151,12 @@ int MPID_Type_struct(int count, int mpi_errno = MPI_SUCCESS; int i, old_are_contig = 1, definitely_not_contig = 0; int found_sticky_lb = 0, found_sticky_ub = 0, found_true_lb = 0, - found_true_ub = 0, found_el_type = 0; + found_true_ub = 0, found_el_type = 0, found_lb=0, found_ub=0; MPI_Aint el_sz = 0; MPI_Aint size = 0; MPI_Datatype el_type = MPI_DATATYPE_NULL; MPI_Aint true_lb_disp = 0, true_ub_disp = 0, sticky_lb_disp = 0, - sticky_ub_disp = 0; + sticky_ub_disp = 0, lb_disp = 0, ub_disp = 0; MPID_Datatype *new_dtp; @@ -320,7 +320,7 @@ int MPID_Type_struct(int count, } } - /* keep lowest true lb and highest true ub + /* keep lowest lb/true_lb and highest ub/true_ub * * note: checking for contiguity at the same time, to avoid * yet another pass over the arrays @@ -339,6 +339,18 @@ int MPID_Type_struct(int count, definitely_not_contig = 1; } + if (!found_lb) + { + found_lb = 1; + lb_disp = tmp_lb; + } + else if (lb_disp > tmp_lb) + { + /* lb before previous */ + lb_disp = tmp_lb; + definitely_not_contig = 1; + } + if (!found_true_ub) { found_true_ub = 1; @@ -352,6 +364,20 @@ int MPID_Type_struct(int count, /* element ends before previous ended */ definitely_not_contig = 1; } + + if (!found_ub) + { + found_ub = 1; + ub_disp = tmp_ub; + } + else if (ub_disp < tmp_ub) + { + ub_disp = tmp_ub; + } + else { + /* ub before previous */ + definitely_not_contig = 1; + } } if (!is_builtin && !old_dtp->is_contig) @@ -366,11 +392,11 @@ int MPID_Type_struct(int count, new_dtp->has_sticky_lb = found_sticky_lb; new_dtp->true_lb = true_lb_disp; - new_dtp->lb = (found_sticky_lb) ? sticky_lb_disp : true_lb_disp; + new_dtp->lb = (found_sticky_lb) ? sticky_lb_disp : lb_disp; new_dtp->has_sticky_ub = found_sticky_ub; new_dtp->true_ub = true_ub_disp; - new_dtp->ub = (found_sticky_ub) ? sticky_ub_disp : true_ub_disp; + new_dtp->ub = (found_sticky_ub) ? sticky_ub_disp : ub_disp; new_dtp->alignsize = MPID_Type_struct_alignsize(count, oldtype_array, ----------------------------------------------------------------------- Summary of changes: src/mpi/datatype/type_create_darray.c | 63 +++++++++++++------- src/mpi/datatype/type_create_subarray.c | 37 +++++------- .../common/datatype/mpid_type_create_resized.c | 8 +- src/mpid/common/datatype/mpid_type_struct.c | 36 ++++++++++-- 4 files changed, 91 insertions(+), 53 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2a2-131-g9a9ffb7
by noreply＠mpich.org 03 Feb '15

03 Feb '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 9a9ffb76c216ae3051e8f02163ba30abf55eeaf0 (commit) from 4d676d9b5b5f709759e0dfcdcc54b9d1e6771174 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/9a9ffb76c216ae3051e8f02163ba30abf… commit 9a9ffb76c216ae3051e8f02163ba30abf55eeaf0 Author: Wesley Bland <wbland(a)anl.gov> Date: Mon Feb 2 17:54:47 2015 -0600 Moves errflag in request from device to top level The errflag in the request object is used in common code and should not have been put in device specific code. This moves it up to the MPI_Request object. Signed-off-by: Sameh Sharkawi <sssharka(a)us.ibm.com> diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h index 90c28e3..4204466 100644 --- a/src/include/mpiimpl.h +++ b/src/include/mpiimpl.h @@ -1509,6 +1509,9 @@ typedef struct MPID_Request { struct MPIR_Sendq *dbg_next; + /* Errflag for NBC requests. Not used by other requests. */ + mpir_errflag_t errflag; + /* Other, device-specific information */ #ifdef MPID_DEV_REQUEST_DECL MPID_DEV_REQUEST_DECL diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 6b379cb..2bb62b2 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -446,9 +446,6 @@ typedef struct MPIDI_Request { struct MPIDI_RMA_Lock_entry *lock_queue_entry; MPI_Request resp_request_handle; /* Handle for get_accumulate response */ - /* Errflag for NBC requests. Not used by other requests. */ - mpir_errflag_t errflag; - MPIDI_REQUEST_SEQNUM /* Occasionally, when a message cannot be sent, we need to cache the diff --git a/src/mpid/ch3/src/ch3u_request.c b/src/mpid/ch3/src/ch3u_request.c index 19c0747..d9bdac0 100644 --- a/src/mpid/ch3/src/ch3u_request.c +++ b/src/mpid/ch3/src/ch3u_request.c @@ -73,6 +73,7 @@ MPID_Request * MPID_Request_create(void) MPIR_STATUS_SET_CANCEL_BIT(req->status, FALSE); req->comm = NULL; req->greq_fns = NULL; + req->errflag = MPIR_ERR_NONE; req->dev.datatype_ptr = NULL; req->dev.segment_ptr = NULL; /* Masks and flags for channel device state in an MPID_Request */ @@ -94,7 +95,6 @@ MPID_Request * MPID_Request_create(void) req->dev.user_buf = NULL; req->dev.final_user_buf = NULL; req->dev.drop_data = FALSE; - req->dev.errflag = MPIR_ERR_NONE; #ifdef MPIDI_CH3_REQUEST_INIT MPIDI_CH3_REQUEST_INIT(req); #endif diff --git a/src/mpid/common/sched/mpid_sched.c b/src/mpid/common/sched/mpid_sched.c index 86027cd..ecda136 100644 --- a/src/mpid/common/sched/mpid_sched.c +++ b/src/mpid/common/sched/mpid_sched.c @@ -140,29 +140,29 @@ static int MPIDU_Sched_start_entry(struct MPIDU_Sched *s, size_t idx, struct MPI * &send.count, but this requires patching up the pointers * during realloc of entries, so this is easier */ ret_errno = MPIC_Isend(e->u.send.buf, *e->u.send.count_p, e->u.send.datatype, - e->u.send.dest, s->tag, comm, &e->u.send.sreq, &r->dev.errflag); + e->u.send.dest, s->tag, comm, &e->u.send.sreq, &r->errflag); } else { if (e->u.send.is_sync) { ret_errno = MPIC_Issend(e->u.send.buf, e->u.send.count, e->u.send.datatype, - e->u.send.dest, s->tag, comm, &e->u.send.sreq, &r->dev.errflag); + e->u.send.dest, s->tag, comm, &e->u.send.sreq, &r->errflag); } else { ret_errno = MPIC_Isend(e->u.send.buf, e->u.send.count, e->u.send.datatype, - e->u.send.dest, s->tag, comm, &e->u.send.sreq, &r->dev.errflag); + e->u.send.dest, s->tag, comm, &e->u.send.sreq, &r->errflag); } } /* Check if the error is actually fatal to the NBC or we can continue. */ if (unlikely(ret_errno)) { - if (MPIR_ERR_NONE == r->dev.errflag) { + if (MPIR_ERR_NONE == r->errflag) { if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(ret_errno)) { - r->dev.errflag = MPIR_ERR_PROC_FAILED; + r->errflag = MPIR_ERR_PROC_FAILED; } else { - r->dev.errflag = MPIR_ERR_OTHER; + r->errflag = MPIR_ERR_OTHER; } } e->status = MPIDU_SCHED_ENTRY_STATUS_FAILED; - MPIU_DBG_MSG_D(COMM, VERBOSE, "Sched SEND failed. Errflag: %d\n", (int) r->dev.errflag); + MPIU_DBG_MSG_D(COMM, VERBOSE, "Sched SEND failed. Errflag: %d\n", (int) r->errflag); } else { e->status = MPIDU_SCHED_ENTRY_STATUS_STARTED; } @@ -174,15 +174,15 @@ static int MPIDU_Sched_start_entry(struct MPIDU_Sched *s, size_t idx, struct MPI e->u.recv.src, s->tag, comm, &e->u.recv.rreq); /* Check if the error is actually fatal to the NBC or we can continue. */ if (unlikely(ret_errno)) { - if (MPIR_ERR_NONE == r->dev.errflag) { + if (MPIR_ERR_NONE == r->errflag) { if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(ret_errno)) { - r->dev.errflag = MPIR_ERR_PROC_FAILED; + r->errflag = MPIR_ERR_PROC_FAILED; } else { - r->dev.errflag = MPIR_ERR_OTHER; + r->errflag = MPIR_ERR_OTHER; } } e->status = MPIDU_SCHED_ENTRY_STATUS_FAILED; - MPIU_DBG_MSG_D(COMM, VERBOSE, "Sched SEND failed. Errflag: %d\n", (int) r->dev.errflag); + MPIU_DBG_MSG_D(COMM, VERBOSE, "Sched SEND failed. Errflag: %d\n", (int) r->errflag); } else { e->status = MPIDU_SCHED_ENTRY_STATUS_STARTED; } @@ -218,11 +218,11 @@ static int MPIDU_Sched_start_entry(struct MPIDU_Sched *s, size_t idx, struct MPI if (e->u.cb.cb_type == MPIDU_SCHED_CB_TYPE_1) { ret_errno = e->u.cb.u.cb_p(r->comm, s->tag, e->u.cb.cb_state); if (unlikely(ret_errno)) { - if (MPIR_ERR_NONE == r->dev.errflag) { + if (MPIR_ERR_NONE == r->errflag) { if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(ret_errno)) { - r->dev.errflag = MPIR_ERR_PROC_FAILED; + r->errflag = MPIR_ERR_PROC_FAILED; } else { - r->dev.errflag = MPIR_ERR_OTHER; + r->errflag = MPIR_ERR_OTHER; } } e->status = MPIDU_SCHED_ENTRY_STATUS_FAILED; @@ -233,11 +233,11 @@ static int MPIDU_Sched_start_entry(struct MPIDU_Sched *s, size_t idx, struct MPI else if (e->u.cb.cb_type == MPIDU_SCHED_CB_TYPE_2) { ret_errno = e->u.cb.u.cb2_p(r->comm, s->tag, e->u.cb.cb_state, e->u.cb.cb_state2); if (unlikely(ret_errno)) { - if (MPIR_ERR_NONE == r->dev.errflag) { + if (MPIR_ERR_NONE == r->errflag) { if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(ret_errno)) { - r->dev.errflag = MPIR_ERR_PROC_FAILED; + r->errflag = MPIR_ERR_PROC_FAILED; } else { - r->dev.errflag = MPIR_ERR_OTHER; + r->errflag = MPIR_ERR_OTHER; } } e->status = MPIDU_SCHED_ENTRY_STATUS_FAILED; @@ -841,7 +841,7 @@ static int MPIDU_Sched_progress_state(struct MPIDU_Sched_state *state, int *made case MPIDU_SCHED_ENTRY_SEND: if (e->u.send.sreq != NULL && MPID_Request_is_complete(e->u.send.sreq)) { MPIU_DBG_MSG_FMT(COMM, VERBOSE, (MPIU_DBG_FDEST, "completed SEND entry %d, sreq=%p\n", (int) i, e->u.send.sreq)); - if (s->req->dev.errflag != MPIR_ERR_NONE) + if (s->req->errflag != MPIR_ERR_NONE) e->status = MPIDU_SCHED_ENTRY_STATUS_FAILED; else e->status = MPIDU_SCHED_ENTRY_STATUS_COMPLETE; @@ -854,9 +854,9 @@ static int MPIDU_Sched_progress_state(struct MPIDU_Sched_state *state, int *made case MPIDU_SCHED_ENTRY_RECV: if (e->u.recv.rreq != NULL && MPID_Request_is_complete(e->u.recv.rreq)) { MPIU_DBG_MSG_FMT(COMM, VERBOSE, (MPIU_DBG_FDEST, "completed RECV entry %d, rreq=%p\n", (int) i, e->u.recv.rreq)); - MPIR_Process_status(&e->u.recv.rreq->status, &s->req->dev.errflag); + MPIR_Process_status(&e->u.recv.rreq->status, &s->req->errflag); MPIR_Request_extract_status(e->u.recv.rreq, e->u.recv.status); - if (s->req->dev.errflag != MPIR_ERR_NONE) + if (s->req->errflag != MPIR_ERR_NONE) e->status = MPIDU_SCHED_ENTRY_STATUS_FAILED; else e->status = MPIDU_SCHED_ENTRY_STATUS_COMPLETE; @@ -894,7 +894,7 @@ static int MPIDU_Sched_progress_state(struct MPIDU_Sched_state *state, int *made MPL_DL_DELETE(state->head, s); /* TODO refactor into a sched_complete routine? */ - switch (s->req->dev.errflag) { + switch (s->req->errflag) { case MPIR_ERR_PROC_FAILED: MPIU_ERR_SET(s->req->status.MPI_ERROR, MPIX_ERR_PROC_FAILED, "**comm"); break; ----------------------------------------------------------------------- Summary of changes: src/include/mpiimpl.h | 3 ++ src/mpid/ch3/include/mpidpre.h | 3 -- src/mpid/ch3/src/ch3u_request.c | 2 +- src/mpid/common/sched/mpid_sched.c | 42 ++++++++++++++++++------------------ 4 files changed, 25 insertions(+), 25 deletions(-) hooks/post-receive -- MPICH primary repository

1 0