September 2014 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.2a1-1-g7944459
by noreply＠mpich.org 05 Sep '14

05 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 79444598bb51599702cabcecab341b5a20847124 (commit) from 0eeed17df88f41285e43c61a4f6863c0c0b85fb3 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/79444598bb51599702cabcecab341b5a2… commit 79444598bb51599702cabcecab341b5a20847124 Author: Igor Ivanov <Igor.Ivanov(a)itseez.com> Date: Thu Sep 4 14:21:10 2014 +0300 netmod/mxm: Cleanup code from warnings Resolved warnings reported by --enable-strict configuration. Signed-off-by: Igor Ivanov <Igor.Ivanov(a)itseez.com> diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_cancel.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_cancel.c index df49239..2de472d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_cancel.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_cancel.c @@ -17,15 +17,17 @@ int MPID_nem_mxm_cancel_send(MPIDI_VC_t * vc, MPID_Request * req) { int mpi_errno = MPI_SUCCESS; mxm_error_t ret = MXM_OK; + MPID_nem_mxm_vc_area *vc_area = VC_BASE(req->ch.vc); + MPID_nem_mxm_req_area *req_area = REQ_BASE(req); _dbg_mxm_output(5, "========> Canceling SEND req %p\n", req); - if (likely(!_mxm_req_test(&REQ_FIELD(req, mxm_req->item.base)))) { - ret = mxm_req_cancel_send(&REQ_FIELD(req, mxm_req->item.send)); + if (likely(!_mxm_req_test(&req_area->mxm_req->item.base))) { + ret = mxm_req_cancel_send(&req_area->mxm_req->item.send); if ((MXM_OK == ret) || (MXM_ERR_NO_PROGRESS == ret)) { - _mxm_req_wait(&REQ_FIELD(req, mxm_req->item.base)); + _mxm_req_wait(&req_area->mxm_req->item.base); if (MPIR_STATUS_GET_CANCEL_BIT(req->status)) { - (VC_FIELD(req->ch.vc, pending_sends)) -= 1; + vc_area->pending_sends -= 1; } } else { @@ -50,13 +52,14 @@ int MPID_nem_mxm_cancel_recv(MPIDI_VC_t * vc, MPID_Request * req) { int mpi_errno = MPI_SUCCESS; mxm_error_t ret = MXM_OK; + MPID_nem_mxm_req_area *req_area = REQ_BASE(req); _dbg_mxm_output(5, "========> Canceling RECV req %p\n", req); - if (likely(!_mxm_req_test(&REQ_FIELD(req, mxm_req->item.base)))) { - ret = mxm_req_cancel_recv(&REQ_FIELD(req, mxm_req->item.recv)); + if (likely(!_mxm_req_test(&req_area->mxm_req->item.base))) { + ret = mxm_req_cancel_recv(&req_area->mxm_req->item.recv); if ((MXM_OK == ret) || (MXM_ERR_NO_PROGRESS == ret)) { - _mxm_req_wait(&REQ_FIELD(req, mxm_req->item.base)); + _mxm_req_wait(&req_area-> mxm_req->item.base); } else { mpi_errno = MPI_ERR_INTERN; diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h index 54d5833..6d3b9b2 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_impl.h @@ -148,9 +148,8 @@ typedef struct { int pending_sends; } MPID_nem_mxm_vc_area; -/* direct macro to private fields in VC */ -#define VC_FIELD(vcp, field) (((MPID_nem_mxm_vc_area *)vcp->ch.netmod_area.padding)->field) -#define VC_BASE(vcp) ((MPID_nem_mxm_vc_area *)vcp->ch.netmod_area.padding) +/* macro for mxm private in VC */ +#define VC_BASE(vcp) ((MPID_nem_mxm_vc_area *)((vcp)->ch.netmod_area.padding)) /* The req provides a generic buffer in which network modules can store private fields This removes all dependencies from the req structure @@ -172,13 +171,12 @@ typedef struct { mxm_req_buffer_t tmp_buf[MXM_MPICH_MAX_IOV]; } MPID_nem_mxm_req_area; -/* direct macro to private fields in REQ */ -#define REQ_FIELD(reqp, field) (((MPID_nem_mxm_req_area *)((reqp)->ch.netmod_area.padding))->field) +/* macro for mxm private in REQ */ #define REQ_BASE(reqp) ((MPID_nem_mxm_req_area *)((reqp)->ch.netmod_area.padding)) typedef struct MPID_nem_mxm_module_t { char *runtime_version; - char *compiletime_version; + const char *compiletime_version; mxm_context_opts_t *mxm_ctx_opts; mxm_ep_opts_t *mxm_ep_opts; mxm_h mxm_context; @@ -331,7 +329,7 @@ static inline void _dbg_mxm_out(int level, } } -static void _dbg_mxm_hexdump(void *ptr, int buflen) +static inline void _dbg_mxm_hexdump(void *ptr, int buflen) { unsigned char *buf = (unsigned char *) ptr; char *str = NULL; @@ -387,7 +385,6 @@ static inline char *_rank_val_to_str(int rank, char *out, int max) static inline void _dbg_mxm_req(MPID_Request * req) { - FILE *stream = stderr; char tag_buf[128]; char rank_buf[128]; diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c index 44283d7..ca31588 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_init.c @@ -92,6 +92,7 @@ MPID_nem_mxm_module_t *mxm_obj; static int _mxm_init(int rank, int size); static int _mxm_fini(void); +static int _mxm_post_init(void); static int _mxm_connect(MPID_nem_mxm_ep_t * ep, const char *business_card, MPID_nem_mxm_vc_area * vc_area); static int _mxm_disconnect(MPID_nem_mxm_ep_t * ep); @@ -101,30 +102,6 @@ static int _mxm_conf(void); #undef FUNCNAME -#define FUNCNAME MPID_nem_mxm_post_init -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_mxm_post_init(void) -{ - int mpi_errno = MPI_SUCCESS; - - _mxm_barrier(); - -#if MXM_API >= MXM_VERSION(3,1) - /* Current logic guarantees that all VCs have been initialized before post init call */ - if (_mxm_obj.conf.bulk_connect) { - mxm_ep_wireup(_mxm_obj.mxm_ep); - } -#endif - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - - -#undef FUNCNAME #define FUNCNAME MPID_nem_mxm_init #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) @@ -153,7 +130,7 @@ int MPID_nem_mxm_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_ if (mpi_errno) MPIU_ERR_POP(mpi_errno); - mpi_errno = MPID_nem_register_initcomp_cb(MPID_nem_mxm_post_init); + mpi_errno = MPID_nem_register_initcomp_cb(_mxm_post_init); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -250,6 +227,7 @@ int MPID_nem_mxm_vc_init(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; MPIDI_CH3I_VC *vc_ch = &vc->ch; + MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); MPIDI_STATE_DECL(MPID_STATE_MXM_VC_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MXM_VC_INIT); @@ -275,9 +253,9 @@ int MPID_nem_mxm_vc_init(MPIDI_VC_t * vc) if (mpi_errno) MPIU_ERR_POP(mpi_errno); - VC_FIELD(vc, ctx) = vc; - VC_FIELD(vc, mxm_ep) = &_mxm_obj.endpoint[vc->pg_rank]; - mpi_errno = _mxm_connect(&_mxm_obj.endpoint[vc->pg_rank], business_card, VC_BASE(vc)); + vc_area->ctx = vc; + vc_area->mxm_ep = &_mxm_obj.endpoint[vc->pg_rank]; + mpi_errno = _mxm_connect(&_mxm_obj.endpoint[vc->pg_rank], business_card, vc_area); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -286,7 +264,7 @@ int MPID_nem_mxm_vc_init(MPIDI_VC_t * vc) MPIDI_CHANGE_VC_STATE(vc, ACTIVE); - VC_FIELD(vc, pending_sends) = 0; + vc_area->pending_sends = 0; vc->rndvSend_fn = NULL; vc->rndvRecv_fn = NULL; @@ -319,8 +297,9 @@ int MPID_nem_mxm_vc_destroy(MPIDI_VC_t * vc) * to destroy endpoint here */ #if 0 - if (VC_FIELD(vc, ctx) == vc) { - mpi_errno = _mxm_disconnect(VC_FIELD(vc, mxm_ep)); + MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); + if (vc_area->ctx == vc) { + mpi_errno = _mxm_disconnect(vc_area->mxm_ep); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } @@ -340,6 +319,7 @@ int MPID_nem_mxm_vc_destroy(MPIDI_VC_t * vc) int MPID_nem_mxm_vc_terminate(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; + MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); MPIDI_STATE_DECL(MPID_STATE_MXM_VC_TERMINATE); MPIDI_FUNC_ENTER(MPID_STATE_MXM_VC_TERMINATE); @@ -351,7 +331,7 @@ int MPID_nem_mxm_vc_terminate(MPIDI_VC_t * vc) MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank); } else { - while ((VC_FIELD(vc, pending_sends)) > 0) + while (vc_area->pending_sends > 0) MPID_nem_mxm_poll(FALSE); } @@ -369,9 +349,7 @@ int MPID_nem_mxm_vc_terminate(MPIDI_VC_t * vc) static int _mxm_conf(void) { int mpi_errno = MPI_SUCCESS; - mxm_error_t ret = MXM_OK; unsigned long cur_ver; - char *env_val = NULL; cur_ver = mxm_get_version(); if (cur_ver != MXM_API) { @@ -514,6 +492,25 @@ static int _mxm_fini(void) goto fn_exit; } +static int _mxm_post_init(void) +{ + int mpi_errno = MPI_SUCCESS; + + _mxm_barrier(); + +#if MXM_API >= MXM_VERSION(3,1) + /* Current logic guarantees that all VCs have been initialized before post init call */ + if (_mxm_obj.conf.bulk_connect) { + mxm_ep_wireup(_mxm_obj.mxm_ep); + } +#endif + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + static int _mxm_connect(MPID_nem_mxm_ep_t * ep, const char *business_card, MPID_nem_mxm_vc_area * vc_area) { @@ -576,8 +573,10 @@ static int _mxm_add_comm(MPID_Comm * comm, void *param) mxm_error_t ret = MXM_OK; mxm_mq_h mxm_mq; - _dbg_mxm_output(6, "Add COMM comm %p (context %d rank %d) \n", - comm, comm->context_id, comm->rank); + _dbg_mxm_output(6, "Add COMM comm %p (rank %d type %d context %d | %d size %d | %d) \n", + comm, comm->rank, comm->comm_kind, + comm->context_id, comm->recvcontext_id, + comm->local_size, comm->remote_size); ret = mxm_mq_create(_mxm_obj.mxm_context, comm->context_id, &mxm_mq); MPIU_ERR_CHKANDJUMP1(ret != MXM_OK, @@ -597,8 +596,8 @@ static int _mxm_del_comm(MPID_Comm * comm, void *param) int mpi_errno = MPI_SUCCESS; mxm_mq_h mxm_mq = (mxm_mq_h) comm->dev.ch.netmod_priv; - _dbg_mxm_output(6, "Del COMM comm %p (context %d rank %d) \n", - comm, comm->context_id, comm->rank); + _dbg_mxm_output(6, "Del COMM comm %p (rank %d type %d) \n", + comm, comm->rank, comm->comm_kind); if (mxm_mq) mxm_mq_destroy(mxm_mq); diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c index eef92ee..87259f3 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_poll.c @@ -105,8 +105,8 @@ void MPID_nem_mxm_anysource_posted(MPID_Request * req) #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_mxm_anysource_matched(MPID_Request * req) { - int mpi_errno = MPI_SUCCESS; mxm_error_t ret = MXM_OK; + MPID_nem_mxm_req_area *req_area = NULL; int matched = FALSE; /* This function is called when an anysource request in the posted @@ -123,12 +123,13 @@ int MPID_nem_mxm_anysource_matched(MPID_Request * req) _dbg_mxm_output(5, "Any Source ========> Matching req %p \n", req); - ret = mxm_req_cancel_recv(&REQ_FIELD(req, mxm_req->item.recv)); + req_area = REQ_BASE(req); + ret = mxm_req_cancel_recv(&req_area->mxm_req->item.recv); if ((MXM_OK == ret) || (MXM_ERR_NO_PROGRESS == ret)) { MPID_Segment_free(req->dev.segment_ptr); } else { - _mxm_req_wait(&REQ_FIELD(req, mxm_req->item.base)); + _mxm_req_wait(&req_area->mxm_req->item.base); matched = TRUE; } @@ -146,8 +147,6 @@ int MPID_nem_mxm_anysource_matched(MPID_Request * req) int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) { int mpi_errno = MPI_SUCCESS; - mxm_error_t ret = MXM_OK; - mxm_recv_req_t *mxm_rreq; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_RECV); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_RECV); @@ -157,16 +156,15 @@ int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) (vc && !vc->ch.is_local)); { - MPIR_Rank_t source = rreq->dev.match.parts.rank; MPIR_Context_id_t context_id = rreq->dev.match.parts.context_id; int tag = rreq->dev.match.parts.tag; - int ret; MPIDI_msg_sz_t data_sz; int dt_contig; MPI_Aint dt_true_lb; MPID_Datatype *dt_ptr; + MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); + MPID_nem_mxm_req_area *req_area = REQ_BASE(rreq); - MPIU_Assert((rreq->kind == MPID_REQUEST_RECV) || (rreq->kind == MPID_PREQUEST_RECV)); MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb); rreq->dev.OnDataAvail = NULL; @@ -177,26 +175,26 @@ int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) "Recv ========> Getting USER msg for req %p (context %d rank %d tag %d size %d) \n", rreq, context_id, source, tag, data_sz); - REQ_FIELD(rreq, ctx) = rreq; - REQ_FIELD(rreq, iov_buf) = REQ_FIELD(rreq, tmp_buf); - REQ_FIELD(rreq, iov_count) = 0; - REQ_FIELD(rreq, iov_buf)[0].ptr = NULL; - REQ_FIELD(rreq, iov_buf)[0].length = 0; + req_area->ctx = rreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; + req_area->iov_buf[0].ptr = NULL; + req_area->iov_buf[0].length = 0; if (dt_contig) { - REQ_FIELD(rreq, iov_count) = 1; - REQ_FIELD(rreq, iov_buf)[0].ptr = (char *) (rreq->dev.user_buf) + dt_true_lb; - REQ_FIELD(rreq, iov_buf)[0].length = data_sz; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (char *) (rreq->dev.user_buf) + dt_true_lb; + req_area->iov_buf[0].length = data_sz; } else { mpi_errno = _mxm_process_rdtype(&rreq, rreq->dev.datatype, dt_ptr, data_sz, rreq->dev.user_buf, rreq->dev.user_count, - &REQ_FIELD(rreq, iov_buf), &REQ_FIELD(rreq, iov_count)); + &req_area->iov_buf, &req_area->iov_count); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } - mpi_errno = _mxm_irecv((vc ? VC_FIELD(vc, mxm_ep) : NULL), REQ_BASE(rreq), + mpi_errno = _mxm_irecv((vc ? vc_area->mxm_ep : NULL), req_area, tag, (rreq->comm ? (mxm_mq_h) rreq->comm->dev.ch.netmod_priv : mxm_obj-> mxm_mq), _mxm_tag_mpi2mxm(tag, context_id)); @@ -217,7 +215,6 @@ int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) static int _mxm_handle_rreq(MPID_Request * req) { - int mpi_errno = MPI_SUCCESS; int complete = FALSE; int dt_contig; MPI_Aint dt_true_lb; @@ -225,6 +222,8 @@ static int _mxm_handle_rreq(MPID_Request * req) MPID_Datatype *dt_ptr; MPIDI_msg_sz_t data_sz; MPIDI_VC_t *vc = NULL; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL MPIU_THREAD_CS_ENTER(MSGQUEUE, req); complete = MPIDI_CH3U_Recvq_DP(req) @@ -236,10 +235,13 @@ static int _mxm_handle_rreq(MPID_Request * req) MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr, dt_true_lb); + vc_area = VC_BASE(req->ch.vc); + req_area = REQ_BASE(req); + _dbg_mxm_output(5, "========> Completing RECV req %p status %d\n", req, req->status.MPI_ERROR); - _dbg_mxm_out_buf(REQ_FIELD(req, iov_buf)[0].ptr, - (REQ_FIELD(req, iov_buf)[0].length > - 16 ? 16 : REQ_FIELD(req, iov_buf)[0].length)); + _dbg_mxm_out_buf(req_area->iov_buf[0].ptr, + (req_area->iov_buf[0].length > + 16 ? 16 : req_area->iov_buf[0].length)); if (req->dev.recv_data_sz <= userbuf_sz) { data_sz = req->dev.recv_data_sz; @@ -282,10 +284,10 @@ static int _mxm_handle_rreq(MPID_Request * req) } } - if (REQ_FIELD(req, iov_count) > MXM_MPICH_MAX_IOV) { - MPIU_Free(REQ_FIELD(req, iov_buf)); - REQ_FIELD(req, iov_buf) = REQ_FIELD(req, tmp_buf); - REQ_FIELD(req, iov_count) = 0; + if (req_area->iov_count > MXM_MPICH_MAX_IOV) { + MPIU_Free(req_area->iov_buf); + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; } MPIDI_CH3U_Handle_recv_req(vc, req, &complete); @@ -299,24 +301,26 @@ static void _mxm_recv_completion_cb(void *context) { MPID_Request *req = (MPID_Request *) context; mxm_recv_req_t *mxm_rreq; + MPID_nem_mxm_req_area *req_area = NULL; MPIU_Assert(req); - MPIU_Assert((req->kind == MPID_REQUEST_RECV) || (req->kind == MPID_PREQUEST_RECV)); _dbg_mxm_out_req(req); - _mxm_to_mpi_status(REQ_FIELD(req, mxm_req->item.base.error), &req->status); + req_area = REQ_BASE(req); + _mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status); - mxm_rreq = &REQ_FIELD(req, mxm_req->item.recv); + mxm_rreq = &req_area->mxm_req->item.recv; req->status.MPI_TAG = _mxm_tag_mxm2mpi(mxm_rreq->completion.sender_tag); req->status.MPI_SOURCE = mxm_rreq->completion.sender_imm; req->dev.recv_data_sz = mxm_rreq->completion.actual_len; MPIR_STATUS_SET_COUNT(req->status, req->dev.recv_data_sz); if (req->ch.vc) { - list_enqueue(&VC_FIELD(req->ch.vc, mxm_ep->free_queue), &REQ_FIELD(req, mxm_req->queue)); + MPID_nem_mxm_vc_area *vc_area = VC_BASE(req->ch.vc); + list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue); } else { - list_enqueue(&mxm_obj->free_queue, &REQ_FIELD(req, mxm_req->queue)); + list_enqueue(&mxm_obj->free_queue, &req_area->mxm_req->queue); } if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) { diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c index 001908a..c5d6d8a 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_probe.c @@ -19,13 +19,14 @@ int MPID_nem_mxm_probe(MPIDI_VC_t * vc, int source, int tag, MPID_Comm * comm, i int mpi_errno = MPI_SUCCESS; mxm_error_t err; mxm_recv_req_t mxm_req; + MPID_nem_mxm_vc_area *vc_area = (vc ? VC_BASE(vc) : NULL); MPIDI_STATE_DECL(MPID_STATE_MXM_PROBE); MPIDI_FUNC_ENTER(MPID_STATE_MXM_PROBE); mxm_req.base.state = MXM_REQ_NEW; mxm_req.base.mq = (mxm_mq_h) comm->dev.ch.netmod_priv; - mxm_req.base.conn = (vc ? VC_FIELD(vc, mxm_ep->mxm_conn) : 0); + mxm_req.base.conn = (vc_area ? vc_area->mxm_ep->mxm_conn : 0); mxm_req.tag = _mxm_tag_mpi2mxm(tag, comm->context_id + context_offset); mxm_req.tag_mask = _mxm_tag_mask(tag); @@ -63,13 +64,14 @@ int MPID_nem_mxm_iprobe(MPIDI_VC_t * vc, int source, int tag, MPID_Comm * comm, int mpi_errno = MPI_SUCCESS; mxm_error_t err; mxm_recv_req_t mxm_req; + MPID_nem_mxm_vc_area *vc_area = (vc ? VC_BASE(vc) : NULL); MPIDI_STATE_DECL(MPID_STATE_MXM_IPROBE); MPIDI_FUNC_ENTER(MPID_STATE_MXM_IPROBE); mxm_req.base.state = MXM_REQ_NEW; mxm_req.base.mq = (mxm_mq_h) comm->dev.ch.netmod_priv; - mxm_req.base.conn = (vc ? VC_FIELD(vc, mxm_ep->mxm_conn) : 0); + mxm_req.base.conn = (vc_area ? vc_area->mxm_ep->mxm_conn : 0); mxm_req.tag = _mxm_tag_mpi2mxm(tag, comm->context_id + context_offset); mxm_req.tag_mask = _mxm_tag_mask(tag); diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c index fe13044..701ed35 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/mxm_send.c @@ -19,10 +19,11 @@ static int _mxm_handle_sreq(MPID_Request * req); static void _mxm_send_completion_cb(void *context); static int _mxm_isend(MPID_nem_mxm_ep_t * ep, MPID_nem_mxm_req_area * req, int type, mxm_mq_h mxm_mq, int mxm_rank, int id, mxm_tag_t tag, int block); +#if 0 /* Consider using this function in case non contiguous data */ static int _mxm_process_sdtype(MPID_Request ** rreq_p, MPI_Datatype datatype, MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, const void *buf, int count, mxm_req_buffer_t ** iov_buf, int *iov_count); - +#endif #undef FUNCNAME #define FUNCNAME MPID_nem_mxm_iSendContig @@ -32,6 +33,8 @@ int MPID_nem_mxm_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MP void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG); @@ -42,27 +45,30 @@ int MPID_nem_mxm_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MP MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t)); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (void *) &(sreq->dev.pending_pkt); - REQ_FIELD(sreq, iov_buf)[0].length = sizeof(MPIDI_CH3_Pkt_t); - if (data_sz) { - REQ_FIELD(sreq, iov_count) = 2; - REQ_FIELD(sreq, iov_buf)[1].ptr = (void *) data; - REQ_FIELD(sreq, iov_buf)[1].length = data_sz; - } - _dbg_mxm_output(5, "iSendContig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n", vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), data_sz); - (VC_FIELD(vc, pending_sends)) += 1; + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area->ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt); + req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t); + if (data_sz) { + req_area->iov_count = 2; + req_area->iov_buf[1].ptr = (void *) data; + req_area->iov_buf[1].length = data_sz; + } + + vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = FALSE; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND_AM, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -84,6 +90,8 @@ int MPID_nem_mxm_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_ { int mpi_errno = MPI_SUCCESS; MPID_Request *sreq = NULL; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG); @@ -106,22 +114,25 @@ int MPID_nem_mxm_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_ vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), data_sz); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (void *) &(sreq->dev.pending_pkt); - REQ_FIELD(sreq, iov_buf)[0].length = sizeof(MPIDI_CH3_Pkt_t); + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area->ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt); + req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t); if (data_sz) { - REQ_FIELD(sreq, iov_count) = 2; - REQ_FIELD(sreq, iov_buf)[1].ptr = (void *) data; - REQ_FIELD(sreq, iov_buf)[1].length = data_sz; + req_area->iov_count = 2; + req_area->iov_buf[1].ptr = (void *) data; + req_area->iov_buf[1].length = data_sz; } - (VC_FIELD(vc, pending_sends)) += 1; + vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = FALSE; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND_AM, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -144,6 +155,8 @@ int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t last; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); @@ -158,11 +171,14 @@ int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), sreq->dev.segment_size); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (void *) &(sreq->dev.pending_pkt); - REQ_FIELD(sreq, iov_buf)[0].length = sizeof(MPIDI_CH3_Pkt_t); + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area->ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt); + req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t); MPIU_Assert(sreq->dev.segment_first == 0); last = sreq->dev.segment_size; @@ -172,16 +188,16 @@ int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.tmpbuf); MPIU_Assert(last == sreq->dev.segment_size); - REQ_FIELD(sreq, iov_count) = 2; - REQ_FIELD(sreq, iov_buf)[1].ptr = sreq->dev.tmpbuf; - REQ_FIELD(sreq, iov_buf)[1].length = last; + req_area->iov_count = 2; + req_area->iov_buf[1].ptr = sreq->dev.tmpbuf; + req_area->iov_buf[1].length = last; } - (VC_FIELD(vc, pending_sends)) += 1; + vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = TRUE; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND_AM, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIU_ERR_POP(mpi_errno); @@ -207,6 +223,8 @@ int MPID_nem_mxm_send(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype int dt_contig; MPIDI_msg_sz_t data_sz; MPI_Aint dt_true_lb; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SEND); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SEND); @@ -233,17 +251,20 @@ int MPID_nem_mxm_send(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype "Send ========> Sending USER msg for req %p (context %d rank %d tag %d size %d) \n", sreq, comm->context_id + context_offset, comm->rank, tag, data_sz); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 0; - REQ_FIELD(sreq, iov_buf)[0].ptr = NULL; - REQ_FIELD(sreq, iov_buf)[0].length = 0; + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area->ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; + req_area->iov_buf[0].ptr = NULL; + req_area->iov_buf[0].length = 0; if (data_sz) { if (dt_contig) { - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (char *) (buf) + dt_true_lb; - REQ_FIELD(sreq, iov_buf)[0].length = data_sz; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb; + req_area->iov_buf[0].length = data_sz; } else { MPIDI_msg_sz_t last; @@ -261,17 +282,17 @@ int MPID_nem_mxm_send(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0); MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = sreq->dev.tmpbuf; - REQ_FIELD(sreq, iov_buf)[0].length = last; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = sreq->dev.tmpbuf; + req_area->iov_buf[0].length = last; } sreq->ch.noncontig = TRUE; } } - (VC_FIELD(vc, pending_sends)) += 1; + vc_area->pending_sends += 1; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND, (mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag, comm->context_id + @@ -304,6 +325,8 @@ int MPID_nem_mxm_ssend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype int dt_contig; MPIDI_msg_sz_t data_sz; MPI_Aint dt_true_lb; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SSEND); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SSEND); @@ -330,17 +353,20 @@ int MPID_nem_mxm_ssend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype "sSend ========> Sending USER msg for req %p (context %d rank %d tag %d size %d) \n", sreq, comm->context_id + context_offset, comm->rank, tag, data_sz); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 0; - REQ_FIELD(sreq, iov_buf)[0].ptr = NULL; - REQ_FIELD(sreq, iov_buf)[0].length = 0; + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area->ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; + req_area->iov_buf[0].ptr = NULL; + req_area->iov_buf[0].length = 0; if (data_sz) { if (dt_contig) { - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (char *) (buf) + dt_true_lb; - REQ_FIELD(sreq, iov_buf)[0].length = data_sz; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb; + req_area->iov_buf[0].length = data_sz; } else { MPIDI_msg_sz_t last; @@ -358,17 +384,17 @@ int MPID_nem_mxm_ssend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0); MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = sreq->dev.tmpbuf; - REQ_FIELD(sreq, iov_buf)[0].length = last; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = sreq->dev.tmpbuf; + req_area->iov_buf[0].length = last; } sreq->ch.noncontig = TRUE; } } - (VC_FIELD(vc, pending_sends)) += 1; + vc_area->pending_sends += 1; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND_SYNC, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC, (mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag, comm->context_id + @@ -401,6 +427,8 @@ int MPID_nem_mxm_isend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype int dt_contig; MPIDI_msg_sz_t data_sz; MPI_Aint dt_true_lb; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISEND); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISEND); @@ -427,17 +455,20 @@ int MPID_nem_mxm_isend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype "iSend ========> Sending USER msg for req %p (context %d rank %d tag %d size %d) \n", sreq, comm->context_id + context_offset, comm->rank, tag, data_sz); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 0; - REQ_FIELD(sreq, iov_buf)[0].ptr = NULL; - REQ_FIELD(sreq, iov_buf)[0].length = 0; + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area->ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; + req_area->iov_buf[0].ptr = NULL; + req_area->iov_buf[0].length = 0; if (data_sz) { if (dt_contig) { - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (char *) (buf) + dt_true_lb; - REQ_FIELD(sreq, iov_buf)[0].length = data_sz; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb; + req_area->iov_buf[0].length = data_sz; } else { MPIDI_msg_sz_t last; @@ -455,17 +486,17 @@ int MPID_nem_mxm_isend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0); MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = sreq->dev.tmpbuf; - REQ_FIELD(sreq, iov_buf)[0].length = last; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = sreq->dev.tmpbuf; + req_area->iov_buf[0].length = last; } sreq->ch.noncontig = TRUE; } } - (VC_FIELD(vc, pending_sends)) += 1; + vc_area->pending_sends += 1; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND, (mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag, comm->context_id + @@ -499,6 +530,8 @@ int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatyp int dt_contig; MPIDI_msg_sz_t data_sz; MPI_Aint dt_true_lb; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISSEND); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISSEND); @@ -525,17 +558,20 @@ int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatyp "isSend ========> Sending USER msg for req %p (context %d rank %d tag %d size %d) \n", sreq, comm->context_id + context_offset, comm->rank, tag, data_sz); - REQ_FIELD(sreq, ctx) = sreq; - REQ_FIELD(sreq, iov_buf) = REQ_FIELD(sreq, tmp_buf); - REQ_FIELD(sreq, iov_count) = 0; - REQ_FIELD(sreq, iov_buf)[0].ptr = NULL; - REQ_FIELD(sreq, iov_buf)[0].length = 0; + vc_area = VC_BASE(vc); + req_area = REQ_BASE(sreq); + + req_area-> ctx = sreq; + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; + req_area->iov_buf[0].ptr = NULL; + req_area->iov_buf[0].length = 0; if (data_sz) { if (dt_contig) { - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = (char *) (buf) + dt_true_lb; - REQ_FIELD(sreq, iov_buf)[0].length = data_sz; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb; + req_area->iov_buf[0].length = data_sz; } else { MPIDI_msg_sz_t last; @@ -553,17 +589,17 @@ int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatyp MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0); MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf); - REQ_FIELD(sreq, iov_count) = 1; - REQ_FIELD(sreq, iov_buf)[0].ptr = sreq->dev.tmpbuf; - REQ_FIELD(sreq, iov_buf)[0].length = last; + req_area->iov_count = 1; + req_area->iov_buf[0].ptr = sreq->dev.tmpbuf; + req_area->iov_buf[0].length = last; } sreq->ch.noncontig = TRUE; } } - (VC_FIELD(vc, pending_sends)) += 1; + vc_area->pending_sends += 1; - mpi_errno = _mxm_isend(VC_FIELD(vc, mxm_ep), REQ_BASE(sreq), MXM_MPICH_ISEND_SYNC, + mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC, (mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag, comm->context_id + @@ -587,21 +623,26 @@ static int _mxm_handle_sreq(MPID_Request * req) { int complete = FALSE; int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; + + vc_area = VC_BASE(req->ch.vc); + req_area = REQ_BASE(req); _dbg_mxm_output(5, "========> Completing SEND req %p status %d\n", req, req->status.MPI_ERROR); - _dbg_mxm_out_buf(REQ_FIELD(req, iov_buf)[0].ptr, - (REQ_FIELD(req, iov_buf)[0].length > - 16 ? 16 : REQ_FIELD(req, iov_buf)[0].length)); + _dbg_mxm_out_buf(req_area->iov_buf[0].ptr, + (req_area->iov_buf[0].length > + 16 ? 16 : req_area->iov_buf[0].length)); - (VC_FIELD(req->ch.vc, pending_sends)) -= 1; + vc_area->pending_sends -= 1; if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))) { MPIU_Free(req->dev.tmpbuf); } - if (REQ_FIELD(req, iov_count) > MXM_MPICH_MAX_IOV) { - MPIU_Free(REQ_FIELD(req, iov_buf)); - REQ_FIELD(req, iov_buf) = REQ_FIELD(req, tmp_buf); - REQ_FIELD(req, iov_count) = 0; + if (req_area->iov_count > MXM_MPICH_MAX_IOV) { + MPIU_Free(req_area->iov_buf); + req_area->iov_buf = req_area->tmp_buf; + req_area->iov_count = 0; } reqFn = req->dev.OnDataAvail; @@ -625,13 +666,18 @@ static int _mxm_handle_sreq(MPID_Request * req) static void _mxm_send_completion_cb(void *context) { MPID_Request *req = (MPID_Request *) context; + MPID_nem_mxm_vc_area *vc_area = NULL; + MPID_nem_mxm_req_area *req_area = NULL; MPIU_Assert(req); _dbg_mxm_out_req(req); - _mxm_to_mpi_status(REQ_FIELD(req, mxm_req->item.base.error), &req->status); + vc_area = VC_BASE(req->ch.vc); + req_area = REQ_BASE(req); - list_enqueue(&VC_FIELD(req->ch.vc, mxm_ep->free_queue), &REQ_FIELD(req, mxm_req->queue)); + _mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status); + + list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue); if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) { _mxm_handle_sreq(req); @@ -718,7 +764,7 @@ static int _mxm_isend(MPID_nem_mxm_ep_t * ep, MPID_nem_mxm_req_area * req, goto fn_exit; } - +#if 0 /* Consider using this function in case non contiguous data */ static int _mxm_process_sdtype(MPID_Request ** sreq_p, MPI_Datatype datatype, MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, const void *buf, int count, mxm_req_buffer_t ** iov_buf, int *iov_count) @@ -791,3 +837,4 @@ static int _mxm_process_sdtype(MPID_Request ** sreq_p, MPI_Datatype datatype, fn_fail: goto fn_exit; } +#endif ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/netmod/mxm/mxm_cancel.c | 17 +- .../ch3/channels/nemesis/netmod/mxm/mxm_impl.h | 13 +- .../ch3/channels/nemesis/netmod/mxm/mxm_init.c | 75 +++--- .../ch3/channels/nemesis/netmod/mxm/mxm_poll.c | 66 +++--- .../ch3/channels/nemesis/netmod/mxm/mxm_probe.c | 6 +- .../ch3/channels/nemesis/netmod/mxm/mxm_send.c | 241 ++++++++++++-------- 6 files changed, 235 insertions(+), 183 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository annotated tag, v3.2a1, created. v3.2a1
by noreply＠mpich.org 04 Sep '14

04 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The annotated tag, v3.2a1 has been created at 8003c4dc0266916f6cdce332b9610e898eaca4e2 (tag) tagging 0eeed17df88f41285e43c61a4f6863c0c0b85fb3 (commit) replaces v3.1.2 tagged by Pavan Balaji on Thu Sep 4 09:48:29 2014 -0500 - Log ----------------------------------------------------------------- tagging v3.2a1 Antonio J. Pena (2): Change buffers in mprobe's test from static to dynamic Fixed hwloc compile warning Antonio Pena Monferrer (6): Fixed datatype displacement use in portals4 netmod Fixed Portals matching masking Fixed accessing to NULL struct in portals receive portals4: Avoid setting error on cancelling a request Added 4 more tests to mprobe Fixed MPI_Probe/MPI/Mprobe/MPI_Mrecv (+I variants) Huiwei Lu (3): Improves random number generation in shfp_fname.c Adds a canceling case for barrier_smp_intra Fixes deadlock in multi-threaded MPI_Comm_idup Igor Ivanov (6): netmod/mxm: Add mxm netmod mpid: Add cancel_recv and cancel_send netmod calls nemesis: Fix memory leak related MPID_nem_register_initcomp_cb netmod/mxm: Update MXM netmod/mxm: Add global synchronization during init/fini netmod/mxm: Fix issue with req->dev.tmpbuf corruption Junchao Zhang (7): Fixed compile warnings in MPI_File_f2c with --disable-romio Changed comments from fixed form C to free form ! Avoid accessing null pointer in MPID_Comm_valid_ptr Fix MPI_Aint casting bugs in F77/90 bindings Add some missing subroutines in the F08 binding Added asynchronous attribute to newcomm of MPI_Comm_idup Added more F08 + TS29113 compiler capability checks Ken Raffenetti (10): portals4: swap arguments to MPIU_Memcpy portals4: correct MPIU_Assert conditional ch3: Call netmod cancel functions portals4: set destination in send request object portals4: implement cancel_recv and anysource_matched add test for canceling anysource recv portals4: use datatype lower bounds for non-contig portals4: get interface limits at init rely on interlib dependencies in compile wrappers testsuite: add library dependencies for tests Masamichi Takagi (3): Increase number of connection command buffer Remove always-true comparison Use HCA when unlocking a lock variable for HCAs Michael Blocksome (2): pamid: do not create memregions for shared memory windows pamid: fix MPI_MODE_NOPRECEDE assertion Min Si (5): Add a zero byte MPI_Win_allocate_shared test Add process number check for SHM communicator. Bug-fix: correct the behavior of flush in exclusively locked epoch. Added a test for flush on shared window. Enabled SHM segments detection in MPI_Win_create Norio Yamaguchi (17): Add a command to notify of outstanding_tx_empty Delete some unnecessary increments Fix some conditional judgments in ib_drain_scq Fix transmission processing of large-message Fix clz after calculating power of two Fix data size in a header Fix the buffer address for send/recv Fix wrong type casting Improve the method of IB-dereg_mr Add queue to store outstanding_tx_empty command Fix the managment of memory area for small size Fix netmod-IB to pass RMA-test Fix build warnings in netmod-IB Fix memory release operation in race-condition Fix memory leak in netmod-IB Fix memory leak in netmod-IB Bug-fix: avoid processing unissued ops at end of synchronization calls Paul Coffman (2): Reduce P2PContig communication with local compute Additional optimization of gpfsmpio_bridgeringagg during MPI_File_open Pavan Balaji (26): Clean up commas. Don't start enums with 0. Remove unnecessary typedef. Allow channel packet handlers in the CH3 structure. Created an enum for packet flags. Revert "mpid: Add cancel_recv and cancel_send netmod calls" White-space cleanup. Merge F90 and F2008 enabling. Added F08 support to the CHANGES file. Fix state declarations in netmods. Move channel specific code into a separate channel structure. Use CVARS instead of env in the MXM netmod. Mellanox patch for a netmod-specific communicator context. Missed including pmi.h in mxm_impl.h Initial HCOLL integration. Move removal of mpic++ to uninstall-local. Cleanup uninstall-local. Advertise OFED IB support. Update version information for the 3.2a1 release. Track external libraries separately instead of in LIBS. Cleanup external and internal libraries. Do not duplicate external and wrapper libs. Move netmod-specific configure options to netmod subconfigures. Update configure comment based on EXTERNAL_LIBS changes. Added a note in the CHANGES file about the MXM netmod. Added release notes for the mxm netmod. Rob Latham (18): generated too-long a value for MPI Info key missing header file for time() routine rma-using tests belong in rma HINDEXED_BLOCK is not quite an INDEXED_BLOCK test case for hindexed_block type shameful but necessary wholesale reformatting clean up bigtype test a bit portability: only BGQ uses the MPIX routines create a big-count HINDEXED type Promote 'len' to something bigger than int deferred open refinement: sync from only aggs systemwide-hints: no need for stat scalable system-hints processing Ask the compiler to warn if stack oddly large clean up printf formatting warnings tighten up function visibility fix compiler warnings and shadowed declarations missed c99-style for-loop in earlier commit Sangmin Seo (2): Add a multi-threaded MPI_Get test. Add dependency check for external libs Su Huang (3): pamid: shared mutex macro updates pamid: shared memory optimization for put operation pamid: MPI_Get coredump in __memcpy_power7 when >1 tasks per node (LE) Wesley Bland (35): Reorganize mpi.h for MPIX err codes Move MPIX_ERR_FAIL_STOP to the MPIX errcode section Move MPIX functions to the end of mpi.h Remove old FT functions Introduce MPICH_ERR_LAST_MPIX Destroy request object before setting it to NULL. Rename error code to MPIX_ERR_PROC_FAILED Add MPIX_ERR_PROC_FAILED_PENDING Return MPIX_ERR_PROC_FAILED_PENDING when appropriate Don't compress and order list of failed procs in PMI Add MPIDI_CH3U_Get_failed_group Add MPIX_Comm_failure_ack/get_acked Fix bug where ANY_SOURCE recv could complete when it shouldn't Remove coll_active field in MPIDI_Comm Add test for anysource handling Add MPI_Comm_revoke Add check for revoked communicator Add MPIX_Comm_shrink functionality Add MPIX_Comm_agree Change MPID_Comm_valid_ptr to optionally ignore revoke Don't check the tag if the coll op completed with error Add some basic resilience to allred_group CH3_ISENDV must correctly set mpi_errno Mark new tests as xfail Add test for multiple sends to alive processes Change the func used to poke the progress engine Mark multi_isendalive as xfail for now. Correctly report the error class in receive queue Fix some problems with get_all_failed at larger scales Fix error case for MPIDI_Request_create_null_rreq Don't put revoked requests into the receive queue Revoke any node aware "subcommunicators" as well Expand search for comm_ptr Silence compiler warning about unused variable Mark shrink test as xfail. Xin Zhao (4): Change default values of CVARs in RMA code. Bug-fix: correct the calculation of SHM base addrs. Free SHM comm for platforms without SHM support Add a failed test for RMA on SHM. artem.v.yalozo (1): Hydra Windows Compilation: Distinguish struct and variable names ----------------------------------------------------------------------- hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-152-g0eeed17
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 0eeed17df88f41285e43c61a4f6863c0c0b85fb3 (commit) from 2e46f3a6b660dba9c1ac8ffd4c1767ef548df077 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/0eeed17df88f41285e43c61a4f6863c0c… commit 0eeed17df88f41285e43c61a4f6863c0c0b85fb3 Author: Huiwei Lu <huiweilu(a)mcs.anl.gov> Date: Tue Jul 15 17:34:26 2014 -0500 Fixes deadlock in multi-threaded MPI_Comm_idup This patch implements the "eager" protocol used in MPI_Comm_idup. Without this patch, test/mpi/threads/comm/comm_idup will crash due to segmentation fault caused by livelock. It uses the same algorithm as multi-threaded MPIR_Get_contextid_sparse_group to avoid deadlock in multi-threaded context_id allocation, but in a nonblocking way. The nonblocking mechanism is explained in the comment of function sched_get_cid_nonblock. Fixes #2069 Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpi/comm/commutil.c b/src/mpi/comm/commutil.c index 8713a2f..60db478 100644 --- a/src/mpi/comm/commutil.c +++ b/src/mpi/comm/commutil.c @@ -833,7 +833,17 @@ fn_fail: } -/* In multi-threaded case, mask_in_used is used to maintain thread safety. In +/* EAGER CONTEXT ID ALLOCATION: Attempt to allocate the context ID during the + * initial synchronization step. If eager protocol fails, threads fall back to + * the base algorithm. + * + * They are used to avoid deadlock in multi-threaded case. In single-threaded + * case, they are not used. + */ +static volatile int eager_nelem = -1; +static volatile int eager_in_use = 0; + +/* In multi-threaded case, mask_in_use is used to maintain thread safety. In * single-threaded case, it is always 0. */ static volatile int mask_in_use = 0; @@ -938,13 +948,6 @@ fn_fail: #else /* MPICH_IS_THREADED is set and true */ -/* EAGER CONTEXT ID ALLOCATION: Attempt to allocate the context ID during the - * initial synchronization step. If eager protocol fails, threads fall back to - * the base algorithm. - */ -static volatile int eager_nelem = -1; -static volatile int eager_in_use = 0; - #undef FUNCNAME #define FUNCNAME MPIR_Get_contextid_sparse #undef FCNAME @@ -1230,6 +1233,8 @@ struct gcn_state { MPIR_Context_id_t *ctx0; MPIR_Context_id_t *ctx1; int own_mask; + int own_eager_mask; + int first_iter; MPID_Comm *comm_ptr; MPID_Comm *comm_ptr_inter; MPID_Sched_t s; @@ -1283,21 +1288,48 @@ static int sched_cb_gcn_allocate_cid(MPID_Comm *comm, int tag, void *state) MPIR_Context_id_t newctxid; MPIU_THREAD_CS_ENTER(CONTEXTID,); - if (st->own_mask) { + if (st->own_eager_mask) { + newctxid = MPIR_Find_and_allocate_context_id(st->local_mask); + if (st->ctx0) + *st->ctx0 = newctxid; + if (st->ctx1) + *st->ctx1 = newctxid; + + st->own_eager_mask = 0; + eager_in_use = 0; + + if (newctxid <= 0) { + /* else we did not find a context id. Give up the mask in case + * there is another thread (with a lower input context id) + * waiting for it. We need to ensure that any other threads + * have the opportunity to run, hence yielding */ + MPIU_THREAD_CS_YIELD(CONTEXTID,); + } + } else if (st->own_mask) { newctxid = MPIR_Find_and_allocate_context_id(st->local_mask); if (st->ctx0) *st->ctx0 = newctxid; if (st->ctx1) *st->ctx1 = newctxid; + + /* reset flags for the next try */ mask_in_use = 0; if (newctxid > 0) { if (lowestContextId == st->comm_ptr->context_id) lowestContextId = MPIR_MAXID; + } else { + /* else we did not find a context id. Give up the mask in case + * there is another thread (with a lower input context id) + * waiting for it. We need to ensure that any other threads + * have the opportunity to run, hence yielding */ + MPIU_THREAD_CS_YIELD(CONTEXTID,); } + } else { + /* As above, force this thread to yield */ + MPIU_THREAD_CS_YIELD(CONTEXTID,); } - MPIU_THREAD_CS_EXIT(CONTEXTID,); if (*st->ctx0 == 0) { /* do not own mask, try again */ @@ -1311,6 +1343,8 @@ static int sched_cb_gcn_allocate_cid(MPID_Comm *comm, int tag, void *state) MPID_SCHED_BARRIER(st->s); } + MPIU_THREAD_CS_EXIT(CONTEXTID,); + /* --BEGIN ERROR HANDLING-- */ /* --END ERROR HANDLING-- */ fn_fail: @@ -1327,18 +1361,40 @@ static int sched_cb_gcn_copy_mask(MPID_Comm *comm, int tag, void *state) struct gcn_state *st = state; MPIU_THREAD_CS_ENTER(CONTEXTID,); - if (st->comm_ptr->context_id < lowestContextId) { - lowestContextId = st->comm_ptr->context_id; - } - if (mask_in_use || (st->comm_ptr->context_id != lowestContextId)) { + if (st->first_iter) { memset(st->local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int)); - st->own_mask = 0; + st->own_eager_mask = 0; + + /* Attempt to reserve the eager mask segment */ + if (!eager_in_use && eager_nelem > 0) { + int i; + for (i = 0; i < eager_nelem; i++) + st->local_mask[i] = context_mask[i]; + + eager_in_use = 1; + st->own_eager_mask = 1; + } + st->first_iter = 0; } else { - MPIU_Memcpy(st->local_mask, context_mask, MPIR_MAX_CONTEXT_MASK * sizeof(uint32_t)); - mask_in_use = 1; - st->own_mask = 1; + if (st->comm_ptr->context_id < lowestContextId) { + lowestContextId = st->comm_ptr->context_id; + } + if (mask_in_use || (st->comm_ptr->context_id != lowestContextId)) { + memset(st->local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int)); + st->own_mask = 0; + } else { + /* Copy safe mask segment to local_mask */ + int i; + for (i = 0; i < eager_nelem; i++) + st->local_mask[i] = 0; + for (i = eager_nelem; i < MPIR_MAX_CONTEXT_MASK; i++) + st->local_mask[i] = context_mask[i]; + + mask_in_use = 1; + st->own_mask = 1; + } } - MPIU_THREAD_CS_ENTER(CONTEXTID,); + MPIU_THREAD_CS_EXIT(CONTEXTID,); mpi_errno = st->comm_ptr->coll_fns->Iallreduce_sched(MPI_IN_PLACE, st->local_mask, MPIR_MAX_CONTEXT_MASK, MPI_UINT32_T, MPI_BAND, st->comm_ptr, st->s); @@ -1354,28 +1410,41 @@ fn_fail: } -/** The multi-threaded nonblocking algorithm of getting a context id +/** Allocating a new context ID collectively over the given communicator in a + * nonblocking way. + * + * The nonblocking mechanism is implemented by inserting MPIDU_Sched_entry to + * the nonblocking collective progress, which is a part of the progress engine. + * It uses a two-level linked list 'all_schedules' to manager all nonblocking + * collective calls: the first level is a linked list of struct MPIDU_Sched; + * and each struct MPIDU_Sched is an array of struct MPIDU_Sched_entry. The + * following four functions are used together to implement the algorithm: + * sched_cb_gcn_copy_mask, sched_cb_gcn_allocate_cid, sched_cb_gcn_bcast and + * sched_get_cid_nonblock. * - * 1. It shares the same global flag 'mask_in_use' with other communicator - * functions to protect access to context_mask. And use CONTEXTID lock to - * protext critical sections. + * The above four functions use the same algorithm as + * MPIR_Get_contextid_sparse_group (multi-threaded version) to allocate a + * context id. The algorithm needs to retry the allocation process in the case + * of conflicts. In MPIR_Get_contextid_sparse_group, it is a while loop. In + * the nonblocking algorithm, 1) new entries are inserted to the end of + * schedule to replace the 'while' loop in MPI_Comm_dup algorithm; 2) all + * arguments passed to sched_get_cid_nonblock are saved to gcn_state in order + * to be called in the future; 3) in sched_cb_gcn_allocate_cid, if the first + * try failed, it will insert sched_cb_gcn_copy_mask to the schedule again. * - * 2. It uses the same algorithm as multithreaded MPI_Comm_dup (multi-threaded - * vertion of MPIR_Get_contextid_sparse_group) to allocate a context id, but in - * a nonblocking way. In the case of conflicts, the algorithm needs to retry - * the allocation process again. In the nonblocking algorithm, 1) new entries - * are inserted to the end of schedule to replace the 'while' loop in - * MPI_Comm_dup algorithm; 2) all arguments passed to sched_get_cid_nonblock - * are saved to gcn_state in order to be called in the future; 3) in - * sched_cb_gcn_allocate_cid, if the first try failed, it will insert - * sched_cb_gcn_copy_mask to the schedule again. + * To ensure thread-safety, it shares the same global flag 'mask_in_use' with + * other communicator functions to protect access to context_mask. And use + * CONTEXTID lock to protect critical sections. * - * 3. There is a subtle difference between INTRACOMM and INTERCOMM when + * There is a subtle difference between INTRACOMM and INTERCOMM when * duplicating a communicator. They needed to be treated differently in * current algorithm. Specifically, 1) when calling sched_get_cid_nonblock, the * parameters are different; 2) updating newcommp->recvcontext_id in * MPIR_Get_intercomm_contextid_nonblock has been moved to sched_cb_gcn_bcast * because this should happen after sched_cb_gcn_allocate_cid has succeed. + * + * To avoid deadlock or livelock, it uses the same eager protocol as + * multi-threaded MPIR_Get_contextid_sparse_group. */ #undef FUNCNAME #define FUNCNAME sched_get_cid_nonblock @@ -1406,6 +1475,14 @@ static int sched_get_cid_nonblock(MPID_Comm *comm_ptr, MPIR_Context_id_t *ctx0, st->s = s; st->gcn_cid_kind = gcn_cid_kind; *(st->ctx0) = 0; + st->own_eager_mask = 0; + st->first_iter = 1; + if (eager_nelem < 0) { + /* Ensure that at least one word of deadlock-free context IDs is + always set aside for the base protocol */ + MPIU_Assert( MPIR_CVAR_CTXID_EAGER_SIZE >= 0 && MPIR_CVAR_CTXID_EAGER_SIZE < MPIR_MAX_CONTEXT_MASK-1 ); + eager_nelem = MPIR_CVAR_CTXID_EAGER_SIZE; + } MPIU_THREAD_CS_EXIT(CONTEXTID,); mpi_errno = MPID_Sched_cb(&sched_cb_gcn_copy_mask, st, s); diff --git a/test/mpi/threads/comm/testlist b/test/mpi/threads/comm/testlist index 6d94573..2dfc780 100644 --- a/test/mpi/threads/comm/testlist +++ b/test/mpi/threads/comm/testlist @@ -3,5 +3,5 @@ dup_leak_test 2 comm_dup_deadlock 4 comm_create_threads 4 comm_create_group_threads 4 -comm_idup 4 mpiversion=3.0 xfail=ticket2069 +comm_idup 4 mpiversion=3.0 ctxidup 4 ----------------------------------------------------------------------- Summary of changes: src/mpi/comm/commutil.c | 143 ++++++++++++++++++++++++++++++--------- test/mpi/threads/comm/testlist | 2 +- 2 files changed, 111 insertions(+), 34 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-151-g2e46f3a
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 2e46f3a6b660dba9c1ac8ffd4c1767ef548df077 (commit) via d94a476666fc6dac1163b9767e79aa9e134b507a (commit) from b58d4baf6c32b8e55faf7b3370a80428811ba21f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/2e46f3a6b660dba9c1ac8ffd4c1767ef5… commit 2e46f3a6b660dba9c1ac8ffd4c1767ef548df077 Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Sep 3 17:18:32 2014 -0500 Added release notes for the mxm netmod. No reviewer. diff --git a/README.vin b/README.vin index c7fcaf5..528164b 100644 --- a/README.vin +++ b/README.vin @@ -473,6 +473,25 @@ shared memory has limits on the size of shared memory segments so using this for Nemesis may limit the number of processes that can be started on a single node. +mxm network module +`````````````````` +The mxm netmod provides support for Mellanox InfiniBand adapters. It +can be built with the following configure option: + + --with-device=ch3:nemesis:mxm + +If your MXM library is installed in a non-standard location, you might +need to help configure find it using the following configure option +(assuming the libraries are present in /path/to/mxm/lib and the +include headers are present in /path/to/mxm/include): + + --with-mxm=/path/to/mxm + +(or) + + --with-mxm-lib=/path/to/mxm/lib + --with-mxm-include=/path/to/mxm/include + ib network module ````````````````` The IB netmod provides support for InfiniBand on x86_64 platforms http://git.mpich.org/mpich.git/commitdiff/d94a476666fc6dac1163b9767e79aa9e1… commit d94a476666fc6dac1163b9767e79aa9e134b507a Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Sep 3 17:15:44 2014 -0500 Added a note in the CHANGES file about the MXM netmod. No reviewer. diff --git a/CHANGES b/CHANGES index e826f2c..d5e888d 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,9 @@ # Fortran 2008 bindings are enabled by default and fully supported. + # Added support for the Mellanox MXM InfiniBand interface. (thanks + to Mellanox for the code contribution). + # Added support for OFED IB on Xeon and Xeon Phi. (thanks to RIKEN and University of Tokyo for the contribution). ----------------------------------------------------------------------- Summary of changes: CHANGES | 3 +++ README.vin | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 0 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-149-gb58d4ba
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via b58d4baf6c32b8e55faf7b3370a80428811ba21f (commit) from 3ea5f3ddd3cc6494b083ddf5fe4707d6fc24e52e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/b58d4baf6c32b8e55faf7b3370a804288… commit b58d4baf6c32b8e55faf7b3370a80428811ba21f Author: Min Si <msi(a)il.is.s.u-tokyo.ac.jp> Date: Thu Aug 28 13:46:29 2014 -0500 Enabled SHM segments detection in MPI_Win_create First, cache every SHM window created by Win_allocate or Win_allocate_shared into a global list, and unlink it in Win_free. Then, when user calls Win_create for a new window, check user specified buffer and comm. Enable local SHM communicaiton in the new window if it matches a cached SHM window. It is noted that all the shared resources are still freed by the original SHM window. Matching a SHM window must satisfy following two conditions: 1. The new node comm is equal to, or a subset of the SHM node comm. (Note that in the other cases where two node comms are overlapped, although the overlapped processes could be logically shared, it is not supported for now. To support this, we need to fist modify the implementation of RMA operations in order to remember shared status per target but not just compare its node_id). 2. The buffer is in the range of the SHM segment across local processes in original SHM window (a contigunous segment is mapped across local processes regardless of whether alloc_shared_noncontig is set). Resolves #2161 Signed-off-by: Xin Zhao <xinzhao3(a)illinois.edu> diff --git a/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h b/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h index 5dd7f36..90aec70 100644 --- a/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h +++ b/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h @@ -10,6 +10,7 @@ #include "mpidimpl.h" #include "mpiu_os_wrappers.h" #include "mpid_nem_generic_queue.h" +#include "mpl_utlist.h" #if defined(HAVE_ASSERT_H) #include <assert.h> @@ -126,4 +127,68 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr); "**pthread_mutex %s", strerror(pt_err)); \ } while (0); + +/* Starting of shared window list */ + +typedef struct MPIDI_SHM_Win { + struct MPIDI_SHM_Win *prev; + struct MPIDI_SHM_Win *next; + MPID_Win *win; +} MPIDI_SHM_Win_t; + +typedef MPIDI_SHM_Win_t *MPIDI_SHM_Wins_list_t; + +extern MPIDI_SHM_Wins_list_t shm_wins_list; + +#define MPIDI_SHM_Wins_next_and_continue(elem) {elem = elem->next; continue;} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_SHM_Wins_append +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_SHM_Wins_append(MPIDI_SHM_Wins_list_t * list, MPID_Win * win) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_SHM_Win_t *tmp_ptr; + MPIU_CHKPMEM_DECL(1); + + /* FIXME: We should use a pool allocator here */ + MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_SHM_Win_t *, sizeof(MPIDI_SHM_Win_t), + mpi_errno, "SHM window entry"); + + tmp_ptr->next = NULL; + tmp_ptr->win = win; + + MPL_DL_APPEND(*list, tmp_ptr); + + fn_exit: + MPIU_CHKPMEM_COMMIT(); + return mpi_errno; + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; +} + +/* Unlink an element from the SHM window list + * + * @param IN list Pointer to the SHM window list + * @param IN elem Pointer to the element to be unlinked + */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_SHM_Wins_unlink +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_SHM_Wins_unlink(MPIDI_SHM_Wins_list_t * list, MPID_Win * shm_win) +{ + MPIDI_SHM_Win_t *elem = NULL; + MPIDI_SHM_Win_t *tmp_elem = NULL; + + MPL_LL_SEARCH_SCALAR(*list, elem, win, shm_win); + if (elem != NULL) { + tmp_elem = elem; + MPL_DL_DELETE(*list, elem); + MPIU_Free(tmp_elem); + } +} + #endif /* !defined(MPICH_MPIDI_CH3_IMPL_H_INCLUDED) */ diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c index 6278879..2bc6951 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c @@ -80,7 +80,10 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) /* free shm_base_addrs that's only used for shared memory windows */ MPIU_Free((*win_ptr)->shm_base_addrs); - if ((*win_ptr)->shm_segment_len > 0) { + /* Only allocate and allocate_shared allocate new shared segments */ + if (((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED || + (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE) && + (*win_ptr)->shm_segment_len > 0) { /* detach from shared memory segment */ mpi_errno = MPIU_SHMW_Seg_detach((*win_ptr)->shm_segment_handle, (char **)&(*win_ptr)->shm_base_addr, (*win_ptr)->shm_segment_len); @@ -91,7 +94,11 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) } /* Free shared process mutex memory region */ - if ((*win_ptr)->shm_mutex && (*win_ptr)->shm_segment_len > 0) { + /* Only allocate and allocate_shared allocate new shared mutex. + * FIXME: it causes unnecessary synchronization when using the same mutex. */ + if (((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED || + (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE) && + (*win_ptr)->shm_mutex && (*win_ptr)->shm_segment_len > 0) { MPID_Comm *node_comm_ptr = NULL; /* When allocating shared memory region segment, we need comm of processes @@ -116,6 +123,12 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) MPIU_SHMW_Hnd_finalize(&(*win_ptr)->shm_mutex_segment_handle); } + /* Unlink from global SHM window list if it is original shared window */ + if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED || + (*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE) { + MPIDI_CH3I_SHM_Wins_unlink(&shm_wins_list, (*win_ptr)); + } + mpi_errno = MPIDI_Win_free(win_ptr); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c index 8049289..e176832 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c @@ -16,9 +16,13 @@ MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather); +MPIDI_SHM_Wins_list_t shm_wins_list; + static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr, void *base_ptr, MPID_Win **win_ptr); +static int MPIDI_CH3I_Win_detect_shm(MPID_Win ** win_ptr); + #undef FUNCNAME #define FUNCNAME MPIDI_CH3_Win_fns_init #undef FCNAME @@ -30,14 +34,220 @@ int MPIDI_CH3_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns) MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3_WIN_FNS_INIT); - if (MPIDI_CH3I_Shm_supported()) + if (MPIDI_CH3I_Shm_supported()) { win_fns->allocate_shm = MPIDI_CH3I_Win_allocate_shm; + win_fns->detect_shm = MPIDI_CH3I_Win_detect_shm; + } MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3_WIN_FNS_INIT); return mpi_errno; } +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_SHM_Wins_match +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int MPIDI_CH3I_SHM_Wins_match(MPID_Win ** win_ptr, MPID_Win ** matched_win, + MPI_Aint ** base_shm_offs_ptr) +{ + int mpi_errno = MPI_SUCCESS; + int i, comm_size; + int node_size, node_rank, shm_node_size; + + MPID_Comm *node_comm_ptr = NULL, *shm_node_comm_ptr = NULL; + int *node_ranks = NULL, *node_ranks_in_shm_node = NULL; + MPID_Group *node_group_ptr = NULL, *shm_node_group_ptr = NULL; + int errflag = FALSE; + MPI_Aint *base_shm_offs; + + MPIDI_SHM_Win_t *elem = shm_wins_list; + *matched_win = NULL; + base_shm_offs = *base_shm_offs_ptr; + + MPIU_CHKLMEM_DECL(2); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SHM_WINS_MATCH); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SHM_WINS_MATCH); + + node_comm_ptr = (*win_ptr)->comm_ptr->node_comm; + MPIU_Assert(node_comm_ptr != NULL); + node_size = node_comm_ptr->local_size; + node_rank = node_comm_ptr->rank; + + comm_size = (*win_ptr)->comm_ptr->local_size; + + MPIU_CHKLMEM_MALLOC(node_ranks, int *, node_size * sizeof(int), mpi_errno, "node_ranks"); + MPIU_CHKLMEM_MALLOC(node_ranks_in_shm_node, int *, node_size * sizeof(int), + mpi_errno, "node_ranks_in_shm_comm"); + + for (i = 0; i < node_size; i++) { + node_ranks[i] = i; + } + + mpi_errno = MPIR_Comm_group_impl(node_comm_ptr, &node_group_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + + while (elem != NULL) { + MPID_Win *shm_win = elem->win; + if (!shm_win) + MPIDI_SHM_Wins_next_and_continue(elem); + + /* Compare node_comm. + * + * Only support shm if new node_comm is equal to or a subset of shm node_comm. + * Shm node_comm == a subset of node_comm is not supported, because it means + * some processes of node_comm cannot be shared, but RMA operation simply checks + * the node_id of a target process for distinguishing shm target. */ + shm_node_comm_ptr = shm_win->comm_ptr->node_comm; + shm_node_size = shm_node_comm_ptr->local_size; + + if (node_size > shm_node_size) + MPIDI_SHM_Wins_next_and_continue(elem); + + mpi_errno = MPIR_Comm_group_impl(shm_win->comm_ptr, &shm_node_group_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + + mpi_errno = MPIR_Group_translate_ranks_impl(node_group_ptr, node_size, + node_ranks, shm_node_group_ptr, + node_ranks_in_shm_node); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + + mpi_errno = MPIR_Group_free_impl(shm_node_group_ptr); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + shm_node_group_ptr = NULL; + + int group_diff = 0; + for (i = 0; i < node_size; i++) { + /* not exist in shm_comm->node_comm */ + if (node_ranks_in_shm_node[i] == MPI_UNDEFINED) { + group_diff = 1; + break; + } + } + if (group_diff) + MPIDI_SHM_Wins_next_and_continue(elem); + + /* Gather the offset of base_addr from all local processes. Match only + * when all of them are included in the shm segment in current shm_win. + * + * Note that this collective call must be called after checking the + * group match in order to guarantee all the local processes can perform + * this call. */ + base_shm_offs[node_rank] = (MPI_Aint) ((*win_ptr)->base) + - (MPI_Aint) (shm_win->shm_base_addr); + mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, + base_shm_offs, 1, MPI_AINT, node_comm_ptr, &errflag); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); + + int base_diff = 0; + for (i = 0; i < comm_size; ++i) { + int i_node_rank = (*win_ptr)->comm_ptr->intranode_table[i]; + if (i_node_rank >= 0) { + MPIU_Assert(i_node_rank < node_size); + + if (base_shm_offs[i_node_rank] < 0 || + base_shm_offs[i_node_rank] + (*win_ptr)->sizes[i] > shm_win->shm_segment_len) { + base_diff = 1; + break; + } + } + } + + if (base_diff) + MPIDI_SHM_Wins_next_and_continue(elem); + + /* Found the first matched shm_win */ + *matched_win = shm_win; + break; + } + + fn_exit: + if (node_group_ptr != NULL) + mpi_errno = MPIR_Group_free_impl(node_group_ptr); + /* Only free it here when group_translate_ranks fails. */ + if (shm_node_group_ptr != NULL) + mpi_errno = MPIR_Group_free_impl(shm_node_group_ptr); + + MPIU_CHKLMEM_FREEALL(); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SHM_WINS_MATCH); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Win_detect_shm +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int MPIDI_CH3I_Win_detect_shm(MPID_Win ** win_ptr) +{ + int mpi_errno = MPI_SUCCESS; + MPID_Win *shm_win_ptr = NULL; + int i, comm_size, node_size; + MPI_Aint *base_shm_offs; + + MPIU_CHKPMEM_DECL(1); + MPIU_CHKLMEM_DECL(1); + MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_WIN_DETECT_SHM); + MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_WIN_DETECT_SHM); + + if ((*win_ptr)->comm_ptr->node_comm == NULL) { + goto fn_exit; + } + + node_size = (*win_ptr)->comm_ptr->node_comm->local_size; + comm_size = (*win_ptr)->comm_ptr->local_size; + + MPIU_CHKLMEM_MALLOC(base_shm_offs, MPI_Aint *, node_size * sizeof(MPI_Aint), + mpi_errno, "base_shm_offs"); + + /* Return the first matched shared window. + * It is noted that the shared windows including all local processes are + * stored in every local process in the same order, hence the first matched + * shared window on every local process should be the same. */ + mpi_errno = MPIDI_CH3I_SHM_Wins_match(win_ptr, &shm_win_ptr, &base_shm_offs); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + if (shm_win_ptr == NULL) + goto fn_exit; + + (*win_ptr)->shm_allocated = TRUE; + MPIU_CHKPMEM_MALLOC((*win_ptr)->shm_base_addrs, void **, + comm_size * sizeof(void *), mpi_errno, "(*win_ptr)->shm_base_addrs"); + + /* Compute the base address of shm buffer on each process. + * shm_base_addrs[i] = my_shm_base_addr + off[i] */ + for (i = 0; i < comm_size; i++) { + int i_node_rank; + i_node_rank = (*win_ptr)->comm_ptr->intranode_table[i]; + if (i_node_rank >= 0) { + MPIU_Assert(i_node_rank < node_size); + + (*win_ptr)->shm_base_addrs[i] = + (void *) ((MPI_Aint) shm_win_ptr->shm_base_addr + base_shm_offs[i_node_rank]); + } + else { + (*win_ptr)->shm_base_addrs[i] = NULL; + } + } + + /* TODO: should we use the same mutex or create a new one ? + * It causes unnecessary synchronization.*/ + (*win_ptr)->shm_mutex = shm_win_ptr->shm_mutex; + (*win_ptr)->RMAFns.Win_free = MPIDI_CH3_SHM_Win_free; + + fn_exit: + MPIU_CHKLMEM_FREEALL(); + MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_WIN_DETECT_SHM); + return mpi_errno; + /* --BEGIN ERROR HANDLING-- */ + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; + /* --END ERROR HANDLING-- */ +} #undef FUNCNAME #define FUNCNAME MPIDI_CH3I_Win_allocate_shm @@ -360,6 +570,9 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info * (*win_ptr)->RMAFns.Win_shared_query = MPIDI_CH3_SHM_Win_shared_query; (*win_ptr)->RMAFns.Win_free = MPIDI_CH3_SHM_Win_free; + /* Cache SHM windows */ + MPIDI_CH3I_SHM_Wins_append(&shm_wins_list, (*win_ptr)); + fn_exit: MPIU_CHKLMEM_FREEALL(); MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_WIN_ALLOCATE_SHM); diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index cbeeaf8..6a9db2d 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1120,6 +1120,7 @@ typedef struct { int (*allocate_shared)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); int (*allocate_shm)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **); int (*create_dynamic)(MPID_Info *, MPID_Comm *, MPID_Win **); + int (*detect_shm)(MPID_Win **); } MPIDI_CH3U_Win_fns_t; extern MPIDI_CH3U_Win_fns_t MPIDI_CH3U_Win_fns; diff --git a/src/mpid/ch3/src/ch3u_win_fns.c b/src/mpid/ch3/src/ch3u_win_fns.c index 06ab46a..9e16ce8 100644 --- a/src/mpid/ch3/src/ch3u_win_fns.c +++ b/src/mpid/ch3/src/ch3u_win_fns.c @@ -132,6 +132,14 @@ int MPIDI_CH3U_Win_create(void *base, MPI_Aint size, int disp_unit, MPID_Info *i mpi_errno = MPIDI_CH3U_Win_create_gather(base, size, disp_unit, info, comm_ptr, win_ptr); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } + if (MPIDI_CH3U_Win_fns.detect_shm != NULL) { + /* Detect if shared buffers are specified for the processes in the + * current node. If so, enable shm RMA.*/ + mpi_errno = MPIDI_CH3U_Win_fns.detect_shm(win_ptr); + if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); + goto fn_exit; + } + fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_WIN_CREATE); return mpi_errno; ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/include/mpidi_ch3_impl.h | 65 ++++++ src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c | 17 ++- src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c | 215 +++++++++++++++++++- src/mpid/ch3/include/mpidimpl.h | 1 + src/mpid/ch3/src/ch3u_win_fns.c | 8 + 5 files changed, 303 insertions(+), 3 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-148-g3ea5f3d
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3ea5f3ddd3cc6494b083ddf5fe4707d6fc24e52e (commit) from 9b51441012556210417c6301348e38b32dc3c35e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3ea5f3ddd3cc6494b083ddf5fe4707d6f… commit 3ea5f3ddd3cc6494b083ddf5fe4707d6fc24e52e Author: Rob Latham <robl(a)mcs.anl.gov> Date: Wed Sep 3 16:24:55 2014 -0500 missed c99-style for-loop in earlier commit No Reviewer diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c index a18b02a..a94c996 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c @@ -149,8 +149,8 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, torusSize[i] = hw.Size[i]; dimTorus[i] = hw.isTorus[i]; proc->numNodesInPartition *= hw.Size[i]; - int baseNum = 1; - for (int j=0;j<i;j++) + int baseNum = 1, j; + for (j=0;j<i;j++) baseNum *= hw.Size[j]; proc->nodeRank += (hw.Coords[i] * baseNum); #ifdef bridgeringaggtrace ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-147-g9b51441
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 9b51441012556210417c6301348e38b32dc3c35e (commit) via dacc232decdea8a0ba2c9576b59a794120b8283c (commit) from 2d2ff83d6be5c168e908a4cd426beea238a86885 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/9b51441012556210417c6301348e38b32… commit 9b51441012556210417c6301348e38b32dc3c35e Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Sun Aug 31 22:14:35 2014 -0500 Additional optimization of gpfsmpio_bridgeringagg during MPI_File_open The code that chose the aggs for gpfsmpio_bridgeringagg was written very inefficiently resulting in a large performance degradation during the MPI_File_open. That code has been rewritten to squash the five torus dimensions into one dimension and to loop over that new representation fewer times. Now there is no performance degradation. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c index 1141bab..240c013 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c @@ -167,6 +167,12 @@ typedef struct int bridge; } sortstruct; +typedef struct +{ + int bridgeRank; + int numAggsAssigned; +} bridgeAggAssignment; + static int intsort(const void *p1, const void *p2) { sortstruct *i1, *i2; @@ -215,91 +221,78 @@ ADIOI_BG_compute_agg_ranklist_serial_do (const ADIOI_BG_ConfInfo_t *confInfo, } else { // aggTotal > 1 - ADIOI_BG_ProcInfo_t *allProcInfoAggNodeList = (ADIOI_BG_ProcInfo_t *) ADIOI_Malloc(confInfo->nProcs * sizeof(ADIOI_BG_ProcInfo_t)); - int allProcInfoAggNodeListSize = 0; - int maxManhattanDistanceToBridge = 0; - - // for ppn > 1, assign minumum rank as agg candidate - for (i=0;i<confInfo->nProcs;i++) { - int addProcToAggNodeList = 1; - for (j=0;j<allProcInfoAggNodeListSize;j++) { - if ((allProcInfoAggNodeList[j].torusCoords[0] == all_procInfo[i].torusCoords[0]) && - (allProcInfoAggNodeList[j].torusCoords[1] == all_procInfo[i].torusCoords[1]) && - (allProcInfoAggNodeList[j].torusCoords[2] == all_procInfo[i].torusCoords[2]) && - (allProcInfoAggNodeList[j].torusCoords[3] == all_procInfo[i].torusCoords[3]) && - (allProcInfoAggNodeList[j].torusCoords[4] == all_procInfo[i].torusCoords[4]) && - addProcToAggNodeList) { - // proc is in the node list, replace if this rank is smaller - addProcToAggNodeList = 0; - - if (allProcInfoAggNodeList[j].rank > all_procInfo[i].rank) - allProcInfoAggNodeList[j] = all_procInfo[i]; - } - } // for j - if (addProcToAggNodeList) { - allProcInfoAggNodeList[allProcInfoAggNodeListSize] = all_procInfo[i]; - if (allProcInfoAggNodeList[allProcInfoAggNodeListSize].manhattanDistanceToBridge > maxManhattanDistanceToBridge) - maxManhattanDistanceToBridge = allProcInfoAggNodeList[allProcInfoAggNodeListSize].manhattanDistanceToBridge; - allProcInfoAggNodeListSize++; - } - } // for i + int currentAggListSize = 0; + int numBridgesWithAggAssignments = 0; + bridgeAggAssignment *aggAssignments = (bridgeAggAssignment *)ADIOI_Malloc(confInfo->numBridgeRanks * sizeof(bridgeAggAssignment)); -#ifdef bridgeringaggtrace - fprintf(stderr,"allProcInfoAggNodeListSize is %d aggTotal is %d\n",allProcInfoAggNodeListSize,aggTotal); -#endif - - int *aggNodeBridgeList = (int *) ADIOI_Malloc (allProcInfoAggNodeListSize * sizeof(int)); // list of all bridge ranks - int *aggNodeBridgeListNum = (int *) ADIOI_Malloc (allProcInfoAggNodeListSize * sizeof(int)); - for (i=0;i<allProcInfoAggNodeListSize;i++) { - aggNodeBridgeList[i] = -1; - aggNodeBridgeListNum[i] = 0; - } - - int aggNodeBridgeListSize = 0; - for (i=0;i<allProcInfoAggNodeListSize;i++) { - int foundBridge = 0; - for (j=0;(j<aggNodeBridgeListSize && !foundBridge);j++) { - if (aggNodeBridgeList[j] == allProcInfoAggNodeList[i].bridgeRank) { - foundBridge = 1; - aggNodeBridgeListNum[i]++; - } - } - if (!foundBridge) { - aggNodeBridgeList[aggNodeBridgeListSize] = allProcInfoAggNodeList[i].bridgeRank; - aggNodeBridgeListNum[aggNodeBridgeListSize] = 1; - aggNodeBridgeListSize++; - } - } + int partitionSize = all_procInfo[0].numNodesInPartition; + int *nodesAssigned = (int *)ADIOI_Malloc(partitionSize * sizeof(int)); + for (i=0;i<partitionSize;i++) + nodesAssigned[i] = 0; - // add aggs based on numAggs per bridge, starting at gpfsmpio_bridgeringagg hops and increasing until numAggs aggs found - int currentAggListSize = 0; - for (i=0;i<aggNodeBridgeListSize;i++) { - int currentBridge = aggNodeBridgeList[i]; int currentNumHops = gpfsmpio_bridgeringagg; - int numAggsAssignedToThisBridge = 0; - while ((numAggsAssignedToThisBridge < numAggs) && (currentNumHops <= maxManhattanDistanceToBridge)) { - for (j=0;j<allProcInfoAggNodeListSize;j++) { - if (allProcInfoAggNodeList[j].bridgeRank == currentBridge) { - if (allProcInfoAggNodeList[j].manhattanDistanceToBridge == currentNumHops) { - aggList[currentAggListSize] = allProcInfoAggNodeList[j].rank; + int allAggsAssigned = 0; + + /* Iterate thru the process infos and select aggregators starting at currentNumHops + away. Increase the currentNumHops until all bridges have numAggs assigned to them. + */ + while (!allAggsAssigned) { + /* track whether any aggs are selected durng this round */ + int startingCurrentAggListSize = currentAggListSize; + int numIterForHopsWithNoAggs = 0; + for (i=0;i<confInfo->nProcs;i++) { + if (all_procInfo[i].manhattanDistanceToBridge == currentNumHops) { + if (nodesAssigned[all_procInfo[i].nodeRank] == 0) { // node is not assigned as an agg yet + int foundBridge = 0; + for (j=0;(j<numBridgesWithAggAssignments && !foundBridge);j++) { + if (aggAssignments[j].bridgeRank == all_procInfo[i].bridgeRank) { + foundBridge = 1; + if (aggAssignments[j].numAggsAssigned < numAggs) { + aggAssignments[j].numAggsAssigned++; + nodesAssigned[all_procInfo[i].nodeRank] = 1; + aggList[currentAggListSize] = all_procInfo[i].rank; + currentAggListSize++; #ifdef bridgeringaggtrace - printf("Assigned agg rank %d at torus coords %u %u %u %u %u to bridge %d at torus coords %u %u %u %u %u at a distance of %d hops\n",allProcInfoAggNodeList[j].rank,allProcInfoAggNodeList[j].torusCoords[0],allProcInfoAggNodeList[j].torusCoords[1],allProcInfoAggNodeList[j].torusCoords[2],allProcInfoAggNodeList[j].torusCoords[3],allProcInfoAggNodeList[j].torusCoords[4], currentBridge, all_procInfo[currentBridge].torusCoords[0], all_procInfo[currentBridge].torusCoords[1], all_procInfo[currentBridge].torusCoords[2], all_procInfo[currentBridge].torusCoords[3], all_procInfo[currentBridge].torusCoords[4],currentNumHops); + printf("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",all_procInfo[i].rank,all_procInfo[i].nodeRank,all_procInfo[i].bridgeRank,currentNumHops); #endif + } + } + } + if (!foundBridge) { + aggAssignments[numBridgesWithAggAssignments].bridgeRank = all_procInfo[i].bridgeRank; + aggAssignments[numBridgesWithAggAssignments].numAggsAssigned = 1; + numBridgesWithAggAssignments++; + nodesAssigned[all_procInfo[i].nodeRank] = 1; + aggList[currentAggListSize] = all_procInfo[i].rank; currentAggListSize++; - numAggsAssignedToThisBridge++; - if (numAggsAssignedToThisBridge >= numAggs) - break; +#ifdef bridgeringaggtrace + printf("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",all_procInfo[i].rank,all_procInfo[i].nodeRank,all_procInfo[i].bridgeRank,currentNumHops); +#endif } } } - currentNumHops++; - } // while - ADIOI_Assert(numAggsAssignedToThisBridge == numAggs); - } // for - - ADIOI_Free(allProcInfoAggNodeList); - ADIOI_Free(aggNodeBridgeList); - ADIOI_Free(aggNodeBridgeListNum); + } + + if (numBridgesWithAggAssignments == confInfo->numBridgeRanks) { + allAggsAssigned = 1; + for (i=0;(i<numBridgesWithAggAssignments && allAggsAssigned);i++) { + if (aggAssignments[i].numAggsAssigned < numAggs) + allAggsAssigned = 0; + } + } + currentNumHops++; + /* If 3 rounds go by without selecting an agg abort to avoid + infinite loop. + */ + if (startingCurrentAggListSize == currentAggListSize) + numIterForHopsWithNoAggs++; + else + numIterForHopsWithNoAggs = 0; + ADIOI_Assert(numIterForHopsWithNoAggs <= 3); + } + + ADIOI_Free(aggAssignments); + ADIOI_Free(nodesAssigned); } // else aggTotal > 1 diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c index 59f2219..a18b02a 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c @@ -136,16 +136,23 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, proc->coreID = hw.coreID; if (gpfsmpio_bridgeringagg > 0) { - for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { - proc->torusCoords[i] = hw.Coords[i]; - } #ifdef bridgeringaggtrace if (rank == 0) fprintf(stderr,"Block dimensions:\n"); #endif + + /* Set the numNodesInPartition and nodeRank for this proc + */ + proc->numNodesInPartition = 1; + proc->nodeRank = 0; for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { torusSize[i] = hw.Size[i]; dimTorus[i] = hw.isTorus[i]; + proc->numNodesInPartition *= hw.Size[i]; + int baseNum = 1; + for (int j=0;j<i;j++) + baseNum *= hw.Size[j]; + proc->nodeRank += (hw.Coords[i] * baseNum); #ifdef bridgeringaggtrace if (rank == 0) fprintf(stderr,"Dimension %d has %d elements wrap-around value is %d\n",i,torusSize[i],dimTorus[i]); diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h index d5f36b1..bcdc61f 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h @@ -35,7 +35,8 @@ typedef struct { node, i.e. psetsize*/ int iamBridge; /* am *I* the bridge rank? */ int __ipad[2]; - unsigned torusCoords[MPIX_TORUS_MAX_DIMS]; /* torus coordinates of node on which this rank resides */ + unsigned nodeRank; /* torus coords converted to an integer for use with gpfsmpio_bridgeringagg */ + unsigned numNodesInPartition; /* number of physical nodes in the job partition */ unsigned manhattanDistanceToBridge; /* number of hops between this rank and the bridge node */ } ADIOI_BG_ProcInfo_t __attribute__((aligned(16))); http://git.mpich.org/mpich.git/commitdiff/dacc232decdea8a0ba2c9576b59a79412… commit dacc232decdea8a0ba2c9576b59a794120b8283c Author: Rob Latham <robl(a)mcs.anl.gov> Date: Wed Sep 3 11:48:33 2014 -0500 fix compiler warnings and shadowed declarations We do not often enough compile with --enable-strict on blue gene No Reviewer diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c index 31ba04b..f4fef37 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c @@ -115,14 +115,14 @@ void ADIOI_GPFS_Open(ADIO_File fd, int *error_code) MPI_Comm_rank(fd->comm, &rank); if ((rank == fd->hints->ranklist[0]) || (fd->comm == MPI_COMM_SELF)) { - struct stat64 gpfs_stat; + struct stat64 gpfs_statbuf; /* Get the (real) underlying file system block size */ - rc = stat64(fd->filename, &gpfs_stat); + rc = stat64(fd->filename, &gpfs_statbuf); if (rc >= 0) { - fd->blksize = gpfs_stat.st_blksize; + fd->blksize = gpfs_statbuf.st_blksize; DBGV_FPRINTF(stderr,"Successful stat '%s'. Blocksize=%ld\n", - fd->filename,gpfs_stat.st_blksize); + fd->filename,gpfs_statbuf.st_blksize); } else { diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index d049340..0e2a1d2 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -572,7 +572,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype } } - ADIO_Offset st_loc_ion, end_loc_ion, needs_gpfs_access_cleanup=0; + ADIO_Offset st_loc_ion=0, end_loc_ion=0, needs_gpfs_access_cleanup=0; #ifdef BGQPLATFORM if (ntimes > 0) { /* only set the gpfs hint if we have io - ie this rank is an aggregator -- otherwise will fail for deferred open */ ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c | 8 +- src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c | 2 +- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c | 145 +++++++++++++-------------- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c | 13 ++- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h | 3 +- 5 files changed, 86 insertions(+), 85 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-145-g2d2ff83
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 2d2ff83d6be5c168e908a4cd426beea238a86885 (commit) via a455e2ce55514d9e1e7eea093528086374bde53d (commit) from 5c373c2a0947290333fb350b5f4718b946230d08 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/2d2ff83d6be5c168e908a4cd426beea23… commit 2d2ff83d6be5c168e908a4cd426beea238a86885 Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Sep 3 12:47:27 2014 -0500 Update configure comment based on EXTERNAL_LIBS changes. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/configure.ac b/configure.ac index 7337da4..2ce0acf 100644 --- a/configure.ac +++ b/configure.ac @@ -269,13 +269,10 @@ PAC_PREFIX_ALL_FLAGS(USER) # as include paths). # # All libraries that are detected by MPICH as needed for some of its -# functionality (such as -lpthread) should be added to LIBS so -# autoconf link tests can use them. Libraries that are built by MPICH -# at make time (and hence are not available for autoconf link tests to -# use), such as OPA and MPL, should be added to WRAPPER_LIBS for -# static builds. All libraries in LIBS are added to WRAPPER_LIBS -# (similarly LDFLAGS are added to WRAPPER_LDFLAGS) as well, at the end -# of configure. +# functionality (such as -lpthread) should be added to EXTERNAL_LIBS +# so executables built within MPICH use them. If inter-library +# dependencies are not supported on the platform, these libraries are +# added to the MPICH wrappers (mpicc and friends) as well. PAC_PREFIX_ALL_FLAGS(WRAPPER) WRAPPER_CFLAGS="$CFLAGS $MPICH_MPICC_FLAGS" WRAPPER_CPPFLAGS="$CPPFLAGS $MPICH_MPICPP_FLAGS" http://git.mpich.org/mpich.git/commitdiff/a455e2ce55514d9e1e7eea09352808637… commit a455e2ce55514d9e1e7eea093528086374bde53d Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Sep 3 10:04:51 2014 -0500 Move netmod-specific configure options to netmod subconfigures. We had some settings in the nemesis subconfigure, which seems misplaced. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 index 2cdcce6..adee81b 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 @@ -17,6 +17,7 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_IB],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:ib]) + PAC_SET_HEADER_LIB_PATH(ib) PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB(dcfa.h,dcfa,ibv_open_device,dcfa_found=yes,dcfa_found=no) PAC_POP_FLAG(LIBS) diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 index 940f591..f2169e5 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 @@ -14,6 +14,7 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_MX],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:mx]) + PAC_SET_HEADER_LIB_PATH(mx) PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB_FATAL(mx, myriexpress.h, myriexpress, mx_finalize) PAC_POP_FLAG(LIBS) diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 index efa3890..28dcdd2 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 @@ -14,6 +14,7 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_MXM],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:mxm]) + PAC_SET_HEADER_LIB_PATH(mxm) PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB_FATAL(mxm, [mxm/api/mxm_api.h], mxm, mxm_get_version) PAC_POP_FLAG(LIBS) diff --git a/src/mpid/ch3/channels/nemesis/netmod/scif/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/scif/subconfigure.m4 index 51c1d6b..7a314fd 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/scif/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/scif/subconfigure.m4 @@ -14,10 +14,10 @@ AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[ ])dnl AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ -# nothing to do for scif right now -dnl AM_COND_IF([BUILD_NEMESIS_NETMOD_SCIF],[ -dnl AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:scif]) -dnl ])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_SCIF,...) +AM_COND_IF([BUILD_NEMESIS_NETMOD_SCIF],[ + AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:scif]) + PAC_SET_HEADER_LIB_PATH(scif) +])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_SCIF,...) ])dnl end _BODY [#] end of __file__ diff --git a/src/mpid/ch3/channels/nemesis/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/subconfigure.m4 index ea46a69..062698f 100644 --- a/src/mpid/ch3/channels/nemesis/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/subconfigure.m4 @@ -91,12 +91,6 @@ This turns off error checking and timing collection],,enable_fast=no) AC_CHECK_HEADERS(signal.h) AC_CHECK_FUNCS(signal) -# Check for netmod relevant headers and libraries -PAC_SET_HEADER_LIB_PATH(mx) -PAC_SET_HEADER_LIB_PATH(scif) -PAC_SET_HEADER_LIB_PATH(ib) -PAC_SET_HEADER_LIB_PATH(mxm) - nemesis_nets_dirs="" nemesis_nets_strings="" nemesis_nets_array="" ----------------------------------------------------------------------- Summary of changes: configure.ac | 11 ++++------- .../ch3/channels/nemesis/netmod/ib/subconfigure.m4 | 1 + .../ch3/channels/nemesis/netmod/mx/subconfigure.m4 | 1 + .../channels/nemesis/netmod/mxm/subconfigure.m4 | 1 + .../channels/nemesis/netmod/scif/subconfigure.m4 | 8 ++++---- src/mpid/ch3/channels/nemesis/subconfigure.m4 | 6 ------ 6 files changed, 11 insertions(+), 17 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-143-g5c373c2
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 5c373c2a0947290333fb350b5f4718b946230d08 (commit) via ae35bf71a2c7b13a3f07bb306f60877efff8887a (commit) via 51db1a05720c3d1a51eff58084a02b6b6c66e80f (commit) from f4518bf66d6c3af9e7a27a6fa9c02ce5f5ede1b2 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/5c373c2a0947290333fb350b5f4718b94… commit 5c373c2a0947290333fb350b5f4718b946230d08 Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Sep 3 00:46:28 2014 -0500 Do not duplicate external and wrapper libs. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/configure.ac b/configure.ac index 9e912e4..7337da4 100644 --- a/configure.ac +++ b/configure.ac @@ -1179,7 +1179,6 @@ else AS_IF([test -s "${with_mpl_prefix}/include/mplconfig.h"], [:],[AC_MSG_ERROR([the MPL installation in "${with_mpl_prefix}" appears broken])]) PAC_APPEND_FLAG([-I${with_mpl_prefix}/include],[CPPFLAGS]) - PAC_PREPEND_FLAG([-l${MPLLIBNAME}],[WRAPPER_LIBS]) PAC_PREPEND_FLAG([-l${MPLLIBNAME}],[EXTERNAL_LIBS]) PAC_APPEND_FLAG([-L${with_mpl_prefix}/lib],[WRAPPER_LDFLAGS]) mpllibdir="${with_mpl_prefix}/lib" @@ -1244,7 +1243,6 @@ if test "$with_openpa_prefix" = "embedded" ; then AC_MSG_WARN([Attempted to use the embedded OpenPA source tree in "src/openpa", but it is missing. Configuration or compilation may fail later.]) fi elif test "$with_openpa_prefix" = "system" ; then - PAC_PREPEND_FLAG([-l${OPALIBNAME}],[WRAPPER_LIBS]) PAC_PREPEND_FLAG([-l${OPALIBNAME}],[EXTERNAL_LIBS]) elif test "$with_openpa_prefix" = "no" ; then # The user doesn't want to use OPA. This may or may not cause MPICH to @@ -1256,7 +1254,6 @@ else AS_IF([test -s "${with_openpa_prefix}/include/opa_primitives.h" -a -s "${with_openpa_prefix}/include/opa_config.h"], [:],[AC_MSG_ERROR([the OpenPA installation in "${with_openpa_prefix}" appears broken])]) PAC_APPEND_FLAG([-I${with_openpa_prefix}/include],[CPPFLAGS]) - PAC_PREPEND_FLAG([-l${OPALIBNAME}],[WRAPPER_LIBS]) PAC_PREPEND_FLAG([-l${OPALIBNAME}],[EXTERNAL_LIBS]) if test -d ${with_openpa_prefix}/lib64 ; then PAC_APPEND_FLAG([-L${with_openpa_prefix}/lib64],[WRAPPER_LDFLAGS]) @@ -5982,7 +5979,7 @@ AC_DEFINE(HAVE_MPICHCONF,1,[Define so that we can test whether the mpichconf.h f # add the LDFLAGS/LIBS we got so far to WRAPPERs if test "$INTERLIB_DEPS" = "no" ; then WRAPPER_LDFLAGS="$WRAPPER_LDFLAGS $LDFLAGS" - WRAPPER_LIBS="$WRAPPER_LIBS $LIBS" + WRAPPER_LIBS="$WRAPPER_LIBS $EXTERNAL_LIBS" fi if test "$USE_PMI2_API" = "yes" ; then http://git.mpich.org/mpich.git/commitdiff/ae35bf71a2c7b13a3f07bb306f60877ef… commit ae35bf71a2c7b13a3f07bb306f60877efff8887a Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Sep 3 00:29:49 2014 -0500 Cleanup external and internal libraries. We were mixing up external libraries, which are prebuilt, and internal libraries on which we need to add dependencies. This patch separates these out and only adds build dependencies on the internal libraries. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/Makefile.am b/Makefile.am index ba184df..4bbe965 100644 --- a/Makefile.am +++ b/Makefile.am @@ -49,9 +49,9 @@ errnames_txt_files = external_subdirs = @mplsrcdir@ @opasrcdir@ external_ldflags = -L@mpllibdir@ -L@opalibdir@ -external_libs = @mpllib@ @opalib@ @EXTERNAL_LIBS@ +external_libs = @EXTERNAL_LIBS@ mpi_convenience_libs = -pmpi_convenience_libs = +pmpi_convenience_libs = @mpllib@ @opalib@ # NOTE on our semi-unconventional usage of DIST_SUBDIRS: # The automake manual recommends thinking of DIST_SUBDIRS as the list of all @@ -151,7 +151,7 @@ lib_LTLIBRARIES += lib/lib@[email protected] lib_lib@PMPILIBNAME@_la_SOURCES = $(mpi_sources) $(mpi_f77_sources) $(mpi_core_sources) lib_lib@PMPILIBNAME@_la_LDFLAGS = $(external_ldflags) $(ABIVERSIONFLAGS) lib_lib@PMPILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) -EXTRA_lib_lib@PMPILIBNAME@_la_DEPENDENCIES = $(external_libs) +EXTRA_lib_lib@PMPILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) # lib@[email protected] might depend on lib@[email protected]. We add them # in that order to lib_LTLIBRARIES so libtool doesn't get @@ -169,7 +169,7 @@ lib_lib@MPILIBNAME@_la_SOURCES = $(mpi_sources) $(mpi_core_sources) lib_lib@MPILIBNAME@_la_LDFLAGS = $(external_ldflags) $(ABIVERSIONFLAGS) lib_lib@MPILIBNAME@_la_CPPFLAGS = $(AM_CPPFLAGS) lib_lib@MPILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) $(mpi_convenience_libs) -EXTRA_lib_lib@MPILIBNAME@_la_DEPENDENCIES = $(external_libs) +EXTRA_lib_lib@MPILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) $(mpi_convenience_libs) endif !BUILD_PROFILING_LIB if BUILD_F77_BINDING diff --git a/configure.ac b/configure.ac index ab78149..9e912e4 100644 --- a/configure.ac +++ b/configure.ac @@ -1180,9 +1180,9 @@ else [:],[AC_MSG_ERROR([the MPL installation in "${with_mpl_prefix}" appears broken])]) PAC_APPEND_FLAG([-I${with_mpl_prefix}/include],[CPPFLAGS]) PAC_PREPEND_FLAG([-l${MPLLIBNAME}],[WRAPPER_LIBS]) + PAC_PREPEND_FLAG([-l${MPLLIBNAME}],[EXTERNAL_LIBS]) PAC_APPEND_FLAG([-L${with_mpl_prefix}/lib],[WRAPPER_LDFLAGS]) mpllibdir="${with_mpl_prefix}/lib" - mpllib="-l${MPLLIBNAME}" fi # OpenPA @@ -1245,7 +1245,7 @@ if test "$with_openpa_prefix" = "embedded" ; then fi elif test "$with_openpa_prefix" = "system" ; then PAC_PREPEND_FLAG([-l${OPALIBNAME}],[WRAPPER_LIBS]) - opalib="-l${OPALIBNAME}" + PAC_PREPEND_FLAG([-l${OPALIBNAME}],[EXTERNAL_LIBS]) elif test "$with_openpa_prefix" = "no" ; then # The user doesn't want to use OPA. This may or may not cause MPICH to # fail to configure/build, depending on many other factors. @@ -1257,7 +1257,7 @@ else [:],[AC_MSG_ERROR([the OpenPA installation in "${with_openpa_prefix}" appears broken])]) PAC_APPEND_FLAG([-I${with_openpa_prefix}/include],[CPPFLAGS]) PAC_PREPEND_FLAG([-l${OPALIBNAME}],[WRAPPER_LIBS]) - opalib="-l${OPALIBNAME}" + PAC_PREPEND_FLAG([-l${OPALIBNAME}],[EXTERNAL_LIBS]) if test -d ${with_openpa_prefix}/lib64 ; then PAC_APPEND_FLAG([-L${with_openpa_prefix}/lib64],[WRAPPER_LDFLAGS]) opalibdir="${with_openpa_prefix}/lib64" http://git.mpich.org/mpich.git/commitdiff/51db1a05720c3d1a51eff58084a02b6b6… commit 51db1a05720c3d1a51eff58084a02b6b6c66e80f Author: Pavan Balaji <balaji(a)anl.gov> Date: Tue Sep 2 21:25:56 2014 -0500 Track external libraries separately instead of in LIBS. The original MPICH code was directly adding external library dependencies into LIBS. This forces configure to use these libraries, thus requiring the user to point to these libraries through LD_LIBRARY_PATH (for shared libraries). This is unnecessary since those libraries are only needed when building the executables and not the rest of MPICH. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/Makefile.am b/Makefile.am index 118005c..ba184df 100644 --- a/Makefile.am +++ b/Makefile.am @@ -49,7 +49,7 @@ errnames_txt_files = external_subdirs = @mplsrcdir@ @opasrcdir@ external_ldflags = -L@mpllibdir@ -L@opalibdir@ -external_libs = @mpllib@ @opalib@ +external_libs = @mpllib@ @opalib@ @EXTERNAL_LIBS@ mpi_convenience_libs = pmpi_convenience_libs = diff --git a/configure.ac b/configure.ac index 18e4868..ab78149 100644 --- a/configure.ac +++ b/configure.ac @@ -284,6 +284,9 @@ WRAPPER_FFLAGS="$FFLAGS $MPICH_MPIF77_FLAGS" WRAPPER_FCFLAGS="$FCFLAGS $MPICH_MPIFC_FLAGS" WRAPPER_LDFLAGS="$MPICH_LDFLAGS" WRAPPER_LIBS="$MPICH_LIBS" +EXTERNAL_LIBS="" +export EXTERNAL_LIBS +AC_SUBST(EXTERNAL_LIBS) # Add MPICHLIB_* to the appropriate flags AC_ARG_VAR(MPICHLIB_CFLAGS, @@ -5507,7 +5510,10 @@ AC_ARG_ENABLE(checkpointing, [AC_HELP_STRING([--enable-checkpointing], [Enable application checkpointing])], [ if test "$enableval" != "no" ; then PAC_SET_HEADER_LIB_PATH(blcr) + PAC_PUSH_FLAG([LIBS]) PAC_CHECK_HEADER_LIB_FATAL(blcr, libcr.h, cr, cr_init) + PAC_APPEND_FLAG([-lblcr],[EXTERNAL_LIBS]) + PAC_POP_FLAG([LIBS]) AC_DEFINE(ENABLE_CHECKPOINTING,1,[Application checkpointing enabled]) fi ], ) diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 index 5026002..2cdcce6 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 @@ -17,13 +17,19 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_IB],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:ib]) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB(dcfa.h,dcfa,ibv_open_device,dcfa_found=yes,dcfa_found=no) + PAC_POP_FLAG(LIBS) if test "${dcfa_found}" = "yes" ; then AC_MSG_NOTICE([libdcfa is going to be linked.]) + PAC_APPEND_FLAG([-ldcfa],[EXTERNAL_LIBS]) else + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB([infiniband/verbs.h],ibverbs,ibv_open_device,ibverbs_found=yes,ibverbs_found=no) + PAC_POP_FLAG(LIBS) if test "${ibverbs_found}" = "yes" ; then AC_MSG_NOTICE([libibverbs is going to be linked.]) + PAC_APPEND_FLAG([-libverbs],[EXTERNAL_LIBS]) else AC_MSG_ERROR([Internal error: neither ibverbs nor dcfa was found]) fi diff --git a/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 index eeecbb1..940f591 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/mx/subconfigure.m4 @@ -14,7 +14,9 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_MX],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:mx]) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB_FATAL(mx, myriexpress.h, myriexpress, mx_finalize) + PAC_POP_FLAG(LIBS) AC_CHECK_HEADER([mx_extensions.h], , [ AC_MSG_ERROR(['mx_extensions.h not found. Are you running a recent version of MX (at least 1.2.1)?']) ]) @@ -31,6 +33,7 @@ AM_COND_IF([BUILD_NEMESIS_NETMOD_MX],[ AC_MSG_ERROR(['MX API version Problem. Are you running a recent version of MX (at least 1.2.1)?']) fi; AC_DEFINE([ENABLE_COMM_OVERRIDES], 1, [define to add per-vc function pointers to override send and recv functions]) + PAC_APPEND_FLAG([-lmx],[EXTERNAL_LIBS]) ])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_MX,...) ])dnl end _BODY diff --git a/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 index 6920697..efa3890 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/mxm/subconfigure.m4 @@ -14,7 +14,9 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_MXM],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:mxm]) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB_FATAL(mxm, [mxm/api/mxm_api.h], mxm, mxm_get_version) + PAC_POP_FLAG(LIBS) AC_CHECK_HEADER([mxm/api/mxm_api.h], , [ AC_MSG_ERROR(['mxm/api/mxm_api.h not found.']) ]) @@ -34,6 +36,7 @@ AM_COND_IF([BUILD_NEMESIS_NETMOD_MXM],[ AC_MSG_ERROR(['MXM API version Problem. Are you running a recent version of MXM (at least 3.0)?']) fi; AC_DEFINE([ENABLE_COMM_OVERRIDES], 1, [define to add per-vc function pointers to override send and recv functions]) + PAC_APPEND_FLAG([-lmxm],[EXTERNAL_LIBS]) ])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_MXM,...) ])dnl end _BODY diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/portals4/subconfigure.m4 index 84464f6..2e617f0 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/subconfigure.m4 @@ -15,7 +15,10 @@ AM_COND_IF([BUILD_NEMESIS_NETMOD_PORTALS4],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:portals4]) PAC_SET_HEADER_LIB_PATH(portals4) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB_FATAL(portals4, portals4.h, portals, PtlInit) + PAC_APPEND_FLAG([-lportals4],[EXTERNAL_LIBS]) + PAC_POP_FLAG(LIBS) AC_DEFINE([ENABLE_COMM_OVERRIDES], 1, [define to add per-vc function pointers to override send and recv functions]) diff --git a/src/mpid/ch3/subconfigure.m4 b/src/mpid/ch3/subconfigure.m4 index 461a884..132fe59 100644 --- a/src/mpid/ch3/subconfigure.m4 +++ b/src/mpid/ch3/subconfigure.m4 @@ -76,7 +76,10 @@ AC_ARG_ENABLE([ftb], [Enable FTB support (default is no)])], [AC_DEFINE([ENABLE_FTB], 1, [Define if FTB is enabled]) PAC_SET_HEADER_LIB_PATH([ftb]) - PAC_CHECK_HEADER_LIB_FATAL([ftb], [libftb.h], [ftb], [FTB_Connect])] + PAC_PUSH_FLAG(LIBS) + PAC_CHECK_HEADER_LIB_FATAL([ftb], [libftb.h], [ftb], [FTB_Connect]) + PAC_APPEND_FLAG([-lftb],[EXTERNAL_LIBS]) + PAC_POP_FLAG(LIBS)] ) AC_ARG_WITH(ch3-rank-bits, [--with-ch3-rank-bits=16/32 Number of bits allocated to the rank field (16 or 32)], diff --git a/src/mpid/common/hcoll/subconfigure.m4 b/src/mpid/common/hcoll/subconfigure.m4 index a55e5dd..0050aa9 100644 --- a/src/mpid/common/hcoll/subconfigure.m4 +++ b/src/mpid/common/hcoll/subconfigure.m4 @@ -2,7 +2,12 @@ AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[ PAC_SET_HEADER_LIB_PATH(hcoll) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB([hcoll/api/hcoll_api.h],[hcoll],[hcoll_init],[have_hcoll=yes],[have_hcoll=no]) + if test "$have_hcoll" = "yes" ; then + PAC_APPEND_FLAG([-lhcoll],[EXTERNAL_LIBS]) + fi + PAC_POP_FLAG(LIBS) AM_CONDITIONAL([BUILD_HCOLL],[test "$have_hcoll" = "yes"]) ])dnl end PREREQ diff --git a/src/pm/hydra/Makefile.am b/src/pm/hydra/Makefile.am index 87b63f1..2958098 100644 --- a/src/pm/hydra/Makefile.am +++ b/src/pm/hydra/Makefile.am @@ -7,7 +7,7 @@ external_subdirs = @mpl_srcdir@ external_dist_subdirs = @mpl_dist_srcdir@ external_ldflags = -L$(top_builddir)/@mpl_srcdir@ -external_libs = -lmpl +external_libs = -lmpl @EXTERNAL_LIBS@ bin_PROGRAMS = noinst_HEADERS = diff --git a/src/pm/hydra/configure.ac b/src/pm/hydra/configure.ac index 1e8b324..bb290c4 100644 --- a/src/pm/hydra/configure.ac +++ b/src/pm/hydra/configure.ac @@ -44,7 +44,9 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) # Reset link flags PAC_RESET_LINK_FLAGS() - +EXTERNAL_LIBS="" +export EXTERNAL_LIBS +AC_SUBST(EXTERNAL_LIBS) PAC_ARG_STRICT @@ -265,10 +267,13 @@ for hydra_bss_name in ${hydra_bss_names}; do hydra_bss_external=true # Check if TM library support is available PAC_SET_HEADER_LIB_PATH(pbs) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB(tm.h, torque, tm_init, have_pbs_launcher=yes, have_pbs_launcher=no) + PAC_POP_FLAG(LIBS) if test "$have_pbs_launcher" = "yes" ; then available_launchers="$available_launchers pbs" + PAC_APPEND_FLAG([-ltorque],[EXTERNAL_LIBS]) fi available_rmks="$available_rmks pbs" ;; @@ -502,11 +507,14 @@ for hydra_ckpointlib in ${hydra_ckpointlibs}; do case "$hydra_ckpointlib" in blcr) PAC_SET_HEADER_LIB_PATH(blcr) + PAC_PUSH_FLAG(LIBS) PAC_CHECK_HEADER_LIB(libcr.h, cr, cr_init, have_blcr=yes, have_blcr=no) + PAC_POP_FLAG(LIBS) # BLCR works if test "$have_blcr" = "yes" ; then AC_DEFINE(HAVE_BLCR,1,[Define if blcr is available]) available_ckpointlibs="$available_ckpointlibs blcr" + PAC_APPEND_FLAG([-lcr],[EXTERNAL_LIBS]) fi ;; *) @@ -673,12 +681,13 @@ fi PAC_PUSH_FLAG([LIBS]) PAC_PREPEND_FLAG([-lpthread],[LIBS]) PAC_SET_HEADER_LIB_PATH(ftb) -PAC_POP_FLAG([LIBS]) PAC_CHECK_HEADER_LIB(libftb.h, ftb, FTB_Connect, have_ftb=yes, have_ftb=no) +PAC_POP_FLAG([LIBS]) if test "$have_ftb" = "yes" ; then # FIXME: We need to test to make sure FTB works correctly PAC_APPEND_FLAG([-lpthread],[LIBS]) AC_DEFINE(ENABLE_FTB,1,[Define if FTB is enabled]) + PAC_APPEND_FLAG([-lftb],[EXTERNAL_LIBS]) fi AM_CONDITIONAL([hydra_have_ftb], [test "${have_ftb}" = "yes"]) ----------------------------------------------------------------------- Summary of changes: Makefile.am | 8 ++++---- configure.ac | 17 ++++++++++------- .../ch3/channels/nemesis/netmod/ib/subconfigure.m4 | 6 ++++++ .../ch3/channels/nemesis/netmod/mx/subconfigure.m4 | 3 +++ .../channels/nemesis/netmod/mxm/subconfigure.m4 | 3 +++ .../nemesis/netmod/portals4/subconfigure.m4 | 3 +++ src/mpid/ch3/subconfigure.m4 | 5 ++++- src/mpid/common/hcoll/subconfigure.m4 | 5 +++++ src/pm/hydra/Makefile.am | 2 +- src/pm/hydra/configure.ac | 13 +++++++++++-- 10 files changed, 50 insertions(+), 15 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1.2-140-gf4518bf
by noreply＠mpich.org 03 Sep '14

03 Sep '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via f4518bf66d6c3af9e7a27a6fa9c02ce5f5ede1b2 (commit) via 22924f357e3a571e7f3e1da7706c92d70dce818e (commit) from f6d32e723fc5d7cd9c3adf3c971e3671b5b2bcaa (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/f4518bf66d6c3af9e7a27a6fa9c02ce5f… commit f4518bf66d6c3af9e7a27a6fa9c02ce5f5ede1b2 Author: Min Si <msi(a)il.is.s.u-tokyo.ac.jp> Date: Sat Aug 23 18:46:13 2014 -0500 Added a test for flush on shared window. Flush should guarantee operations are finished on both origin and target side. However, flush may return before the completion on target side in MPI implementation. It makes an error in this case: P0 and P1 allocate a shared window, and P2 locks both of them; P2 first put to P0 and flush, then get the updated data from P1. The put may complete on P0 after the completion of get on P1. Signed-off-by: Xin Zhao <xinzhao3(a)illinois.edu> Signed-off-by: Antonio J. Pena <apenya(a)mcs.anl.gov> diff --git a/test/mpi/rma/Makefile.am b/test/mpi/rma/Makefile.am index 99f928a..ac88fdc 100644 --- a/test/mpi/rma/Makefile.am +++ b/test/mpi/rma/Makefile.am @@ -112,6 +112,7 @@ noinst_PROGRAMS = \ get_accumulate_double_derived \ get_accumulate_int_derived \ flush \ + win_shared_put_flush_get \ reqops \ req_example \ req_example_shm \ diff --git a/test/mpi/rma/testlist.in b/test/mpi/rma/testlist.in index 9e9d3b7..267e041 100644 --- a/test/mpi/rma/testlist.in +++ b/test/mpi/rma/testlist.in @@ -110,6 +110,7 @@ badrma 2 mpiversion=3.0 acc-loc 4 fence_shm 2 mpiversion=3.0 win_shared_zerobyte 4 mpiversion=3.0 +win_shared_put_flush_get 4 mpiversion=3.0 get-struct 2 ## This test is not strictly correct. This was meant to test out the diff --git a/test/mpi/rma/win_shared_put_flush_get.c b/test/mpi/rma/win_shared_put_flush_get.c new file mode 100644 index 0000000..15f1c70 --- /dev/null +++ b/test/mpi/rma/win_shared_put_flush_get.c @@ -0,0 +1,199 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2014 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <mpi.h> +#include "mpitest.h" + +#define ITER 100 +#define BUF_CNT 4 +double local_buf[BUF_CNT], check_buf[BUF_CNT]; + +const int verbose = 0; + +int main(int argc, char *argv[]) +{ + int rank, nproc, i, x; + int errors = 0, all_errors = 0; + MPI_Win win = MPI_WIN_NULL; + + MPI_Comm shm_comm = MPI_COMM_NULL; + int shm_nproc, shm_rank; + double **shm_bases = NULL, *my_base; + MPI_Win shm_win = MPI_WIN_NULL; + MPI_Group shm_group = MPI_GROUP_NULL, world_group = MPI_GROUP_NULL; + int *shm_ranks = NULL, *shm_ranks_in_world = NULL; + MPI_Aint get_target_base_offsets = 0; + + int win_size = sizeof(double) * BUF_CNT; + int new_win_size = win_size; + int win_unit = sizeof(double); + int shm_root_rank_in_world; + int origin = -1, put_target, get_target; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_group(MPI_COMM_WORLD, &world_group); + + if (nproc != 4) { + if (rank == 0) + printf("Error: must be run with four processes\n"); + MPI_Abort(MPI_COMM_WORLD, 1); + } + + MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &shm_comm); + MPI_Comm_rank(shm_comm, &shm_rank); + MPI_Comm_size(shm_comm, &shm_nproc); + MPI_Comm_group(shm_comm, &shm_group); + + /* Platform does not support shared memory or wrong host file, just return. */ + if (shm_nproc != 2) { + goto exit; + } + + shm_bases = (double **) calloc(shm_nproc, sizeof(double *)); + shm_ranks = (int *) calloc(shm_nproc, sizeof(int)); + shm_ranks_in_world = (int *) calloc(shm_nproc, sizeof(int)); + + if (shm_rank == 0) + shm_root_rank_in_world = rank; + MPI_Bcast(&shm_root_rank_in_world, 1, MPI_INT, 0, shm_comm); + + /* Identify ranks of target processes which are located on node 0 */ + if (rank == 0) { + for (i = 0; i < shm_nproc; i++) { + shm_ranks[i] = i; + } + MPI_Group_translate_ranks(shm_group, shm_nproc, shm_ranks, world_group, shm_ranks_in_world); + } + MPI_Bcast(shm_ranks_in_world, shm_nproc, MPI_INT, 0, MPI_COMM_WORLD); + + put_target = shm_ranks_in_world[shm_nproc - 1]; + get_target = shm_ranks_in_world[0]; + + /* Identify the rank of origin process which are located on node 1 */ + if (shm_root_rank_in_world == 1 && shm_rank == 0) { + origin = rank; + if (verbose) { + printf("---- I am origin = %d, get_target = %d, put_target = %d\n", + origin, get_target, put_target); + } + } + + /* Allocate shared memory among local processes */ + MPI_Win_allocate_shared(win_size, win_unit, MPI_INFO_NULL, shm_comm, &my_base, &shm_win); + + if (shm_root_rank_in_world == 0 && verbose) { + MPI_Aint size; + int disp_unit; + for (i = 0; i < shm_nproc; i++) { + MPI_Win_shared_query(shm_win, i, &size, &disp_unit, &shm_bases[i]); + printf("%d -- shared query: base[%d]=%p, size %ld, " + "unit %d\n", rank, i, shm_bases[i], size, disp_unit); + } + } + + /* Get offset of put target(1) on get target(0) */ + get_target_base_offsets = (shm_nproc - 1) * win_size / win_unit; + + if (origin == rank && verbose) + printf("%d -- base_offset of put_target %d on get_target %d: %ld\n", + rank, put_target, get_target, get_target_base_offsets); + + /* Create using MPI_Win_create(). Note that new window size of get_target(0) + * is equal to the total size of shm segments on this node, thus get_target + * process can read the byte located on put_target process.*/ + for (i = 0; i < BUF_CNT; i++) { + local_buf[i] = (i + 1) * 1.0; + my_base[i] = 0.0; + } + + if (get_target == rank) + new_win_size = win_size * shm_nproc; + + MPI_Win_create(my_base, new_win_size, win_unit, MPI_INFO_NULL, MPI_COMM_WORLD, &win); + + if (verbose) + printf("%d -- new window my_base %p, size %ld\n", rank, my_base, new_win_size); + + MPI_Barrier(MPI_COMM_WORLD); + + /* Check if flush guarantees the completion of put operations on target side. + * + * P exclusively locks 2 processes whose windows are shared with each other. + * P first put and flush to a process, then get the updated data from another process. + * If flush returns before operations are done on the target side, the data may be + * incorrect.*/ + for (x = 0; x < ITER; x++) { + for (i = 0; i < BUF_CNT; i++) { + local_buf[i] += x; + check_buf[i] = 0; + } + + if (rank == origin) { + MPI_Win_lock(MPI_LOCK_EXCLUSIVE, put_target, 0, win); + MPI_Win_lock(MPI_LOCK_EXCLUSIVE, get_target, 0, win); + + for (i = 0; i < BUF_CNT; i++) { + MPI_Put(&local_buf[i], 1, MPI_DOUBLE, put_target, i, 1, MPI_DOUBLE, win); + } + MPI_Win_flush(put_target, win); + + MPI_Get(check_buf, BUF_CNT, MPI_DOUBLE, get_target, + get_target_base_offsets, BUF_CNT, MPI_DOUBLE, win); + MPI_Win_flush(get_target, win); + + for (i = 0; i < BUF_CNT; i++) { + if (check_buf[i] != local_buf[i]) { + printf("%d(iter %d) - Got check_buf[%d] = %.1lf, expected %.1lf\n", + rank, x, i, check_buf[i], local_buf[i]); + errors++; + } + } + + MPI_Win_unlock(put_target, win); + MPI_Win_unlock(get_target, win); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + + exit: + + if (rank == 0 && all_errors == 0) + printf(" No Errors\n"); + + if (shm_bases) + free(shm_bases); + if (shm_ranks) + free(shm_ranks); + if (shm_ranks_in_world) + free(shm_ranks_in_world); + + if (shm_win != MPI_WIN_NULL) + MPI_Win_free(&shm_win); + + if (win != MPI_WIN_NULL) + MPI_Win_free(&win); + + if (shm_comm != MPI_COMM_NULL) + MPI_Comm_free(&shm_comm); + + if (shm_group != MPI_GROUP_NULL) + MPI_Group_free(&shm_group); + + if (world_group != MPI_GROUP_NULL) + MPI_Group_free(&world_group); + + MPI_Finalize(); + + return 0; +} http://git.mpich.org/mpich.git/commitdiff/22924f357e3a571e7f3e1da7706c92d70… commit 22924f357e3a571e7f3e1da7706c92d70dce818e Author: Min Si <msi(a)il.is.s.u-tokyo.ac.jp> Date: Sat Aug 23 19:08:29 2014 -0500 Bug-fix: correct the behavior of flush in exclusively locked epoch. FLUSH should guarantee the completion of operations on both origin and target side. However, for exclusive lock, there is an optimization in MPICH which allows FLUSH to return without waiting for the acknowledgement of remote completion from the target side. It relys on the fact that there will be no other processes accessing the window during the exclusive lock epoch. However, such optimization is not correct when two processes allocating windows on overlapping SHM region. Suppose P0 and P1 (on the same node) allocate RMA window using the same SHM region, and P2 (on a different node) locks both windows. P2 first issues a PUT and FLUSH to P0, then issues a GET to P1 on the same memory location with PUT, since FLUSH does not guarantee the remote completion of PUT, GET operation may not get the updated value. This patch disables the optimization for FLUSH and forces FLUSH to always wait for the remote completion of operations. Signed-off-by: Xin Zhao <xinzhao3(a)illinois.edu> Signed-off-by: Antonio J. Pena <apenya(a)mcs.anl.gov> diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index b3c5342..97ab0c8 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -3601,11 +3601,15 @@ static int do_passive_target_rma(MPID_Win *win_ptr, int target_rank, win_ptr->targets[target_rank].remote_lock_assert & MPI_MODE_NOCHECK)); if (win_ptr->targets[target_rank].remote_lock_mode == MPI_LOCK_EXCLUSIVE && - win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_CALLED) { + win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_CALLED && + win_ptr->targets[target_rank].remote_lock_state != MPIDI_CH3_WIN_LOCK_FLUSH) { /* Exclusive lock already held -- no need to wait for rma done pkt at the end. This is because the target won't grant another process access to the window until all of our operations complete at that - target. Thus, there is no third-party communication issue. */ + target. Thus, there is no third-party communication issue. + However, flush still needs to wait for rma done, otherwise result + may be unknown if user reads the updated location from a shared window of + another target process after this flush. */ *wait_for_rma_done_pkt = 0; } else if (MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->targets[target_rank].rma_ops_list)) { ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/src/ch3u_rma_sync.c | 8 +- test/mpi/rma/Makefile.am | 1 + test/mpi/rma/testlist.in | 1 + test/mpi/rma/win_shared_put_flush_get.c | 199 +++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 test/mpi/rma/win_shared_put_flush_get.c hooks/post-receive -- MPICH primary repository

1 0