May 2013 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.0.4-186-g4919577
by noreply＠mpich.org 13 May '13

13 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 491957795644bf6d8edc76f8ee4cd75bf02b7d5c (commit) via b9232780088eadae2d7f00737371e0e43828bfff (commit) via fbcf94a1da1b7e27626d8eb3cba85be8ac1eeb12 (commit) via ae6451d561584d559a0418a455bb690c1c5f237d (commit) from 4325f45932a64cf6b170192f7d4b3ca35924537c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/491957795644bf6d8edc76f8ee4cd75bf… commit 491957795644bf6d8edc76f8ee4cd75bf02b7d5c Author: Su Huang <suhuang(a)us.ibm.com> Date: Fri Sep 21 16:56:03 2012 -0400 multi_mpi coredump with RDMA on mpich (ibm) D186334 (ibm) 7Z6 (ibm) 63716b02ccab7cd99116e927a210eca43b03d52e Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_win_put.c b/src/mpid/pamid/src/onesided/mpid_win_put.c index 8864721..92fb477 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_put.c +++ b/src/mpid/pamid/src/onesided/mpid_win_put.c @@ -65,31 +65,20 @@ MPIDI_Put_use_pami_rput(pami_context_t context, MPIDI_Win_request * req,int *fre { pami_result_t rc; void *map; - - pami_rput_simple_t params = { - .rma = { - .dest = req->dest, - .hints = { - .buffer_registered = PAMI_HINT_ENABLE, - .use_rdma = PAMI_HINT_ENABLE, - }, - .bytes = 0, - .cookie = req, - .done_fn = NULL, - }, - .rdma = { - .local = { - .mr = &req->origin.memregion, - }, - .remote = { - .mr = &req->win->mpid.info[req->target.rank].memregion, - .offset = req->offset, - }, - }, - .put = { - .rdone_fn = MPIDI_Win_DoneCB, - }, - }; + pami_rput_simple_t params; + /* params need to zero out to avoid passing garbage to PAMI */ + memset((void *) &params,0,sizeof(pami_rput_simple_t)); + + params.rma.dest=req->dest; + params.rma.hints.buffer_registered = PAMI_HINT_ENABLE; + params.rma.hints.use_rdma = PAMI_HINT_ENABLE; + params.rma.bytes = 0; + params.rma.cookie = req; + params.rma.done_fn = NULL; + params.rdma.local.mr=&req->origin.memregion; + params.rdma.remote.mr=&req->win->mpid.info[req->target.rank].memregion; + params.rdma.remote.offset= req->offset; + params.put.rdone_fn= MPIDI_Win_DoneCB; struct MPIDI_Win_sync* sync = &req->win->mpid.sync; TRACE_ERR("Start index=%u/%d l-addr=%p r-base=%p r-offset=%zu (sync->started=%u sync->complete=%u)\n", http://git.mpich.org/mpich.git/commitdiff/b9232780088eadae2d7f00737371e0e43… commit b9232780088eadae2d7f00737371e0e43828bfff Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Mon May 13 13:02:10 2013 -0500 Add dropped change from 'code review fixes'. (ibm) D185675 (ibm) 7693f538360130f4cca71f0975c0bb345d2f6239 diff --git a/src/mpid/pamid/include/mpidi_util.h b/src/mpid/pamid/include/mpidi_util.h index 5bec9c6..5428c10 100644 --- a/src/mpid/pamid/include/mpidi_util.h +++ b/src/mpid/pamid/include/mpidi_util.h @@ -42,6 +42,7 @@ #endif #endif +extern int MPIDI_atoi(char* , unsigned int* ); extern int MPIDI_Banner(char *); typedef enum {IS_IP, IS_US} MPIDI_TransportType; typedef enum {NO_STRIPING=0, IS_PACKET_STRIPING} MPIDI_StripingType; http://git.mpich.org/mpich.git/commitdiff/fbcf94a1da1b7e27626d8eb3cba85be8a… commit fbcf94a1da1b7e27626d8eb3cba85be8ac1eeb12 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Mon May 13 12:30:13 2013 -0500 MPIDI_TRACE tool enhancement - remaining updates For some reason these changes from the original commit did not migrate into the 'mpich/master' branch. (ibm) D185675 (ibm) 3728f432f99fc499955a2c9d830a5026e5f87948 diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 7e33c76..2ec80d4 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -285,7 +285,6 @@ struct MPIDI_Request uint32_t shm:16; #endif #ifdef MPIDI_TRACE - int cur_nMsgs; int partner_id; int idx; int PR_idx; diff --git a/src/mpid/pamid/src/mpid_recvq.c b/src/mpid/pamid/src/mpid_recvq.c index 8897296..8a7f43b 100644 --- a/src/mpid/pamid/src/mpid_recvq.c +++ b/src/mpid/pamid/src/mpid_recvq.c @@ -317,7 +317,7 @@ MPIDI_Recvq_FDU(int source, pami_task_t pami_source, int tag, int context_id, in MPIDI_Recvq_remove(MPIDI_Recvq.unexpected, rreq, prev_rreq); found = TRUE; #ifdef MPIDI_TRACE - MPIDI_In_cntr[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ2=1; + MPIDI_Trace_buf[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ2=1; #endif goto fn_exit; } diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c index 5136715..c063bbc 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c @@ -304,8 +304,8 @@ MPIDI_RecvCB(pami_context_t context, recv->addr = rreq->mpid.uebuf; } #ifdef MPIDI_TRACE - MPIDI_In_cntr[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].comp_in_HH=2; - MPIDI_In_cntr[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; + MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].comp_in_HH=2; + MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; #endif #ifdef OUT_OF_ORDER_HANDLING diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c index 296890e..3ec4228 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c @@ -112,10 +112,10 @@ MPIDI_RecvRzvCB_impl(pami_context_t context, #endif rreq->mpid.envelope.length = envelope->length; #ifdef MPIDI_TRACE - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].req=rreq; - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].rzv=1; - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].rlen=envelope->length; - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].sync=msginfo->isSync; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].req=rreq; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].rzv=1; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].rlen=envelope->length; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].sync=msginfo->isSync; #endif if ((TOKEN_FLOW_CONTROL_ON) && (MPIDI_MUST_RETURN_TOKENS(sender))) { @@ -149,9 +149,9 @@ MPIDI_RecvRzvCB_impl(pami_context_t context, { MPIDI_RendezvousTransfer(context, rreq); #ifdef MPIDI_TRACE - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].sync_com_in_HH=1; - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].matchedInHH=1; - MPIDI_In_cntr[source].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].sync_com_in_HH=1; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].matchedInHH=1; + MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; #endif } MPID_Request_discard(newreq); diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c index 2b2aed6..73fd42d 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c @@ -208,9 +208,9 @@ MPIDI_RecvShortCB(pami_context_t context, if (sndlen > 0) memcpy(rcvbuf, sndbuf, sndlen); #ifdef MPIDI_TRACE - MPIDI_In_cntr[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].rlen=sndlen; - MPIDI_In_cntr[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].comp_in_HH=1; - MPIDI_In_cntr[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; + MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].rlen=sndlen; + MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].comp_in_HH=1; + MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; #endif MPIDI_Request_complete(rreq); diff --git a/src/mpid/pamid/src/pt2pt/mpidi_control.c b/src/mpid/pamid/src/pt2pt/mpidi_control.c index 9cc8a2b..b39fa51 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_control.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_control.c @@ -229,7 +229,7 @@ MPIDI_RzvAck_proc_req(pami_context_t context, } #endif #ifdef MPIDI_TRACE - MPIDI_Out_cntr[(req->mpid.partner_id)].S[(req->mpid.idx)].recvAck=1; + MPIDI_Trace_buf[(req->mpid.partner_id)].S[(req->mpid.idx)].recvAck=1; #endif MPIDI_SendDoneCB(context, req, PAMI_SUCCESS); diff --git a/src/mpid/pamid/src/pt2pt/mpidi_recv.h b/src/mpid/pamid/src/pt2pt/mpidi_recv.h index 7042594..2ca6879 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_recv.h +++ b/src/mpid/pamid/src/pt2pt/mpidi_recv.h @@ -160,13 +160,6 @@ MPIDI_Recv(void * buf, tag, comm->recvcontext_id + context_offset, &found); -#ifdef MPIDI_TRACE -{ - size_t ll; - ll = count * MPID_Datatype_get_basic_size(datatype); - SET_REC_PR(rreq,buf,count,ll,datatype,pami_source,rank,tag,comm,is_blocking); -} -#endif #else int pami_source; if(rank != MPI_ANY_SOURCE) { @@ -184,6 +177,13 @@ MPIDI_Recv(void * buf, comm->recvcontext_id + context_offset, &found); #endif +#ifdef MPIDI_TRACE +{ + size_t ll; + ll = count * MPID_Datatype_get_basic_size(datatype); + MPIDI_SET_PR_REC(rreq,buf,count,ll,datatype,pami_source,rank,tag,comm,is_blocking); +} +#endif /* ----------------------------------------------------------------- */ /* populate request with our data */ diff --git a/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c index 0a9eb9d..e070efd 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c @@ -37,7 +37,7 @@ MPIDI_RecvMsg_Unexp(MPID_Request * rreq, /* Recv functions will ack the messages that are unexpected */ /* ------------------------------------------------------------ */ #ifdef MPIDI_TRACE - MPIDI_In_cntr[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ=1; + MPIDI_Trace_buf[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ=1; #endif if (MPIDI_Request_isRzv(rreq)) http://git.mpich.org/mpich.git/commitdiff/ae6451d561584d559a0418a455bb690c1… commit ae6451d561584d559a0418a455bb690c1c5f237d Author: Charles Archer <archerc(a)us.ibm.com> Date: Thu Oct 4 12:01:00 2012 -0400 Fixed assert on int env vars passed in a string Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c index f32fccd..65dc493 100644 --- a/src/mpid/pamid/src/mpidi_env.c +++ b/src/mpid/pamid/src/mpidi_env.c @@ -427,6 +427,7 @@ ENV_Unsigned__(char* name[], unsigned* val, char* string, unsigned num_supported ENV_Deprecated(name, num_supported, deprecated, rank, NA); char * env; + int rc; unsigned i=0; for (;; ++i) { @@ -437,7 +438,17 @@ ENV_Unsigned__(char* name[], unsigned* val, char* string, unsigned num_supported break; } - *val = atoi(env); + unsigned oldval = *val; + rc=MPIDI_atoi(env,val); + if(rc != 0) + { + /* Something went wrong with the processing this integer + * Print a warning, and restore the original value */ + *val = oldval; + fprintf(stderr, "Warning: Environment variable: %s should be an integer value: defaulting to %d", string, *val); + return; + } + if (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL && rank == 0) fprintf(stderr, "%s = %u\n", string, *val); } @@ -932,83 +943,83 @@ MPIDI_Env_setup(int rank, int requested) unsigned tmp; { char* names[] = {"PAMID_COLLECTIVE_ALLGATHER", "PAMI_ALLGATHER", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLGATHER_PREALLREDUCE", "PAMI_ALLGATHER_PREALLREDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLGATHERV", "PAMI_ALLGATHERV", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLGATHERV_PREALLREDUCE", "PAMI_ALLGATHERV_PREALLREDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLREDUCE", "PAMI_ALLREDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLREDUCE_PREALLREDUCE", "PAMI_ALLREDUCE_PREALLREDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLTOALL", "PAMI_ALLTOALL", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLTOALLV", "PAMI_ALLTOALLV", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_ALLTOALLV_INT", "PAMI_ALLTOALLV_INT", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_BARRIER", "PAMI_BARRIER", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_BCAST", "PAMI_BCAST", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_BCAST_PREALLREDUCE", "PAMI_BCAST_PREALLREDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_GATHER", "PAMI_GATHER", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_GATHERV", "PAMI_GATHERV", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_REDUCE", "PAMI_REDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_SCAN", "PAMI_SCAN", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_SCATTER", "PAMI_SCATTER", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_SCATTERV", "PAMI_SCATTERV", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_COLLECTIVE_SCATTERV_PREALLREDUCE", "PAMI_SCATTERV_PREALLREDUCE", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } { char* names[] = {"PAMID_CORE_ON_ABORT", "PAMI_COREONABORT", "PAMI_COREONMPIABORT", "PAMI_COREONMPIDABORT", NULL}; - ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank); + ENV_Deprecated(names, 1, &found_deprecated_env_var, rank, 0); } #if (MPIDI_STATISTICS || MPIDI_PRINTENV) mpich_env=(MPIDI_printenv_t *) MPIU_Malloc(sizeof(MPIDI_printenv_t)+1); ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_datatypes.h | 1 - src/mpid/pamid/include/mpidi_util.h | 1 + src/mpid/pamid/src/mpid_recvq.c | 2 +- src/mpid/pamid/src/mpidi_env.c | 53 ++++++++++++++--------- src/mpid/pamid/src/onesided/mpid_win_put.c | 37 ++++++---------- src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c | 4 +- src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c | 14 +++--- src/mpid/pamid/src/pt2pt/mpidi_callback_short.c | 6 +- src/mpid/pamid/src/pt2pt/mpidi_control.c | 2 +- src/mpid/pamid/src/pt2pt/mpidi_recv.h | 14 +++--- src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c | 2 +- 11 files changed, 68 insertions(+), 68 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-182-g4325f45
by noreply＠mpich.org 10 May '13

10 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 4325f45932a64cf6b170192f7d4b3ca35924537c (commit) via 788dca4a79230a822af8610fe055ea9cc2511b47 (commit) from a811897209cc3087b4a4379438be31884a637b21 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/4325f45932a64cf6b170192f7d4b3ca35… commit 4325f45932a64cf6b170192f7d4b3ca35924537c Author: Haizhu Liu <haizhu(a)us.ibm.com> Date: Thu Apr 25 11:11:05 2013 -0400 Dynamic tasking MPI_Comm_disconnect HANG (ibm) D190081 (ibm) 25939d9ffd5267261f70e714aaec434d84e07093 Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/dyntask/mpidi_port.c b/src/mpid/pamid/src/dyntask/mpidi_port.c index dffc327..7bd2806 100644 --- a/src/mpid/pamid/src/dyntask/mpidi_port.c +++ b/src/mpid/pamid/src/dyntask/mpidi_port.c @@ -35,7 +35,7 @@ typedef struct { conn_info *_conn_info_list = NULL; extern int mpidi_dynamic_tasking; -long long _global_world_intercomm_cntr; +long long _global_world_intercomm_cntr = 0; typedef struct MPIDI_Acceptq { @@ -571,7 +571,7 @@ int MPIDI_Comm_connect(const char *port_name, MPID_Info *info, int root, TRACE_ERR("MPIC_Sendrecv returned with mpi_errno=%d\n", mpi_errno); } - mpi_errno = MPIC_Sendrecv_replace(&comm_cntr, 1, MPI_INT, 0, + mpi_errno = MPIC_Sendrecv_replace(&comm_cntr, 1, MPI_LONG_LONG_INT, 0, sendtag++, 0, recvtag++, tmp_comm->handle, MPI_STATUS_IGNORE); if (mpi_errno != MPI_SUCCESS) { @@ -1226,7 +1226,7 @@ int MPIDI_Comm_accept(const char *port_name, MPID_Info *info, int root, TRACE_ERR("MPIC_Sendrecv returned with mpi_errno=%d\n", mpi_errno); } #endif - mpi_errno = MPIC_Sendrecv_replace(&comm_cntr, 1, MPI_INT, 0, + mpi_errno = MPIC_Sendrecv_replace(&comm_cntr, 1, MPI_LONG_LONG_INT, 0, sendtag++, 0, recvtag++, tmp_comm->handle, MPI_STATUS_IGNORE); if (mpi_errno != MPI_SUCCESS) { http://git.mpich.org/mpich.git/commitdiff/788dca4a79230a822af8610fe055ea9cc… commit 788dca4a79230a822af8610fe055ea9cc2511b47 Author: Haizhu Liu <haizhu(a)us.ibm.com> Date: Mon Apr 22 23:31:55 2013 -0400 MPI_Comm_disconnect hang Also remove MPIR_Comm_group_ext() function declaration and reference. (ibm) D189966 (ibm) 5681895bcb45798680b249dbf993b9a634954b80 Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c b/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c index 3e5378e..1b84f53 100644 --- a/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c +++ b/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c @@ -207,13 +207,16 @@ int MPID_Comm_disconnect(MPID_Comm *comm_ptr) pami_task_t *leader_tids; int expected_firstAM=0, expected_secondAM=0, expected_lastAM=0; MPID_Comm *commworld_ptr; + MPID_Group *group_ptr = NULL, *new_group_ptr = NULL; MPID_VCR *glist; MPID_Comm *lcomm; + int *ranks; int local_tasks=0, localtasks_in_remglist=0; int jobIdSize=64; char jobId[jobIdSize]; int MY_TASKID = PAMIX_Client_query(MPIDI_Client, PAMI_CLIENT_TASK_ID ).value.intval; + /*if( (comm_ptr->comm_kind == MPID_INTERCOMM) && (comm_ptr->mpid.world_ids != NULL)) { */ if(comm_ptr->mpid.world_ids != NULL) { rc = MPID_Iprobe(comm_ptr->rank, MPI_ANY_TAG, comm_ptr, MPID_CONTEXT_INTER_PT2PT, &probe_flag, &status); if(rc || probe_flag) { @@ -223,7 +226,15 @@ int MPID_Comm_disconnect(MPID_Comm *comm_ptr) /* make commSubWorld */ { + /* MPID_Comm_get_ptr( MPI_COMM_WORLD, commworld_ptr ); */ commworld_ptr = MPIR_Process.comm_world; + mpi_errno = MPIR_Comm_group_impl(commworld_ptr, &group_ptr); + if (mpi_errno) + { + TRACE_ERR("Error while creating group_ptr from MPI_COMM_WORLD in MPIDI_Comm_create_from_pami_geom\n"); + return PAMI_ERROR; + } + glist = commworld_ptr->vcr; gsize = commworld_ptr->local_size; @@ -250,25 +261,48 @@ int MPID_Comm_disconnect(MPID_Comm *comm_ptr) } } k=0; - local_list = MPIU_Malloc(local_tasks*sizeof(pami_task_t)); + /* local_list = MPIU_Malloc(local_tasks*sizeof(pami_task_t)); */ + ranks = MPIU_Malloc(local_tasks*sizeof(int)); for(i=0;i<comm_ptr->local_size;i++) { for(j=0;j<gsize;j++) { if(comm_ptr->local_vcr[i]->taskid == glist[j]->taskid) - local_list[k++] = glist[j]->taskid; + /* local_list[k] = glist[j]->taskid; */ + ranks[k++] = j; } } if((comm_ptr->comm_kind == MPID_INTERCOMM) && localtasks_in_remglist) { for(i=0;i<comm_ptr->remote_size;i++) { for(j=0;j<gsize;j++) { if(comm_ptr->vcr[i]->taskid == glist[j]->taskid) - local_list[k++] = glist[j]->taskid; + /* local_list[k] = glist[j]->taskid; */ + ranks[k++] = j; } } /* Sort the local_list when there are localtasks_in_remglist */ - _qsort_dyntask(local_list, 0, local_tasks-1); +/* _qsort_dyntask(local_list, 0, local_tasks-1); */ + _qsort_dyntask(ranks, 0, local_tasks-1); } + + /* Now we have all we need to create the new group. Create it */ + /* mpi_errno = MPIR_Group_incl_impl(group_ptr, local_tasks, ranks, &new_group_ptr); */ + mpi_errno = MPIR_Group_incl_impl(group_ptr, local_tasks, ranks, &new_group_ptr); + if (mpi_errno) + { + TRACE_ERR("Error while creating new_group_ptr from group_ptr in MPIDI_Comm_create_from_pami_geom\n"); + return PAMI_ERROR; + } + /* Now create the communicator using the new_group_ptr */ + mpi_errno = MPIR_Comm_create_group(commworld_ptr, new_group_ptr, 0, &lcomm); + /* mpi_errno = MPIR_Comm_create_intra(commworld_ptr, new_group_ptr, &lcomm); */ + if (mpi_errno) + { + TRACE_ERR("Error while creating new_comm_ptr from group_ptr in MPIDI_Comm_create_from_pami_geom\n"); + return PAMI_ERROR; + } + +#if 0 mpi_errno = MPIR_Comm_create(&lcomm); if (mpi_errno != MPI_SUCCESS) { TRACE_ERR("MPIR_Comm_create returned with mpi_errno=%d\n", mpi_errno); @@ -310,6 +344,8 @@ int MPID_Comm_disconnect(MPID_Comm *comm_ptr) if(MY_TASKID == local_list[i]) lcomm->rank = i; lcomm->vcr[i]->taskid = local_list[i]; } +#endif + } TRACE_ERR("subcomm for disconnect is established local_tasks=%d calling MPIR_Barrier_intra\n", local_tasks); @@ -403,7 +439,8 @@ int MPID_Comm_disconnect(MPID_Comm *comm_ptr) MPIDI_free_tranid_node(comm_ptr->mpid.world_intercomm_cntr); mpi_errno = MPIR_Comm_release(comm_ptr,1); if (mpi_errno) TRACE_ERR("MPIR_Comm_release returned with mpi_errno=%d\n", mpi_errno); - MPIU_Free(local_list); + /* MPIU_Free(local_list); */ + MPIU_Free(ranks); } return mpi_errno; } diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 44e0137..8925c48 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -1003,7 +1003,9 @@ MPIDI_VCRT_init(int rank, int size, char *world_tasks, MPIDI_PG_t *pg) TRACE_ERR("Adding ref for comm=%x vcr=%x pg=%x\n", comm, comm->vcr[0], comm->vcr[0]->pg); MPIDI_PG_add_ref(comm->vcr[0]->pg); } + comm->local_vcr = comm->vcr; } + #endif /* -------------------------------- */ @@ -1051,8 +1053,7 @@ MPIDI_VCRT_init(int rank, int size, char *world_tasks, MPIDI_PG_t *pg) TRACE_ERR("TASKID -- comm->vcr[%d]=%d\n", p, comm->vcr[p]->taskid); } - i = 0; - + comm->local_vcr = comm->vcr; }else { for (i=0; i<size; i++) { comm->vcr[i]->taskid = i; ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c | 47 ++++++++++++++++++-- src/mpid/pamid/src/dyntask/mpidi_port.c | 6 +- src/mpid/pamid/src/mpid_init.c | 5 +- 3 files changed, 48 insertions(+), 10 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-180-ga811897
by noreply＠mpich.org 09 May '13

09 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via a811897209cc3087b4a4379438be31884a637b21 (commit) from faff4d1d5c504ca29b4fafe18978ad3c05e0c6df (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/a811897209cc3087b4a4379438be31884… commit a811897209cc3087b4a4379438be31884a637b21 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Thu May 9 16:21:17 2013 -0500 Make MPIR_Comm_create_group visible to other MPICH utilities. No reviewer. diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h index deedd53..bd0db7c 100644 --- a/src/include/mpiimpl.h +++ b/src/include/mpiimpl.h @@ -4073,6 +4073,8 @@ int MPIR_Reduce_local_impl(const void *inbuf, void *inoutbuf, int count, MPI_Dat int MPIR_Setup_intercomm_localcomm( MPID_Comm * ); int MPIR_Comm_create( MPID_Comm ** ); +int MPIR_Comm_create_group(MPID_Comm * comm_ptr, MPID_Group * group_ptr, int tag, + MPID_Comm ** newcomm); /* comm_create helper functions, used by both comm_create and comm_create_group */ int MPIR_Comm_create_calculate_mapping(MPID_Group *group_ptr, diff --git a/src/mpi/comm/comm_create_group.c b/src/mpi/comm/comm_create_group.c index 9478361..ab946c9 100644 --- a/src/mpi/comm/comm_create_group.c +++ b/src/mpi/comm/comm_create_group.c @@ -18,11 +18,6 @@ #endif /* -- End Profiling Symbol Block */ -/* PMPI_LOCAL should be dropped and this prototype moved to mpiimpl.h if we ever - * need to use this routine outside of this translation unit */ -PMPI_LOCAL int MPIR_Comm_create_group(MPID_Comm * comm_ptr, MPID_Group * group_ptr, int tag, - MPID_Comm ** newcomm); - /* Define MPICH_MPI_FROM_PMPI if weak symbols are not supported to build the MPI routines */ @@ -36,8 +31,8 @@ PMPI_LOCAL int MPIR_Comm_create_group(MPID_Comm * comm_ptr, MPID_Group * group_p #define FCNAME MPIU_QUOTE(FUNCNAME) /* comm create group impl; assumes that the standard error checking * has already taken place in the calling function */ -PMPI_LOCAL int MPIR_Comm_create_group(MPID_Comm * comm_ptr, MPID_Group * group_ptr, int tag, - MPID_Comm ** newcomm_ptr) +int MPIR_Comm_create_group(MPID_Comm * comm_ptr, MPID_Group * group_ptr, int tag, + MPID_Comm ** newcomm_ptr) { int mpi_errno = MPI_SUCCESS; MPIR_Context_id_t new_context_id = 0; ----------------------------------------------------------------------- Summary of changes: src/include/mpiimpl.h | 2 ++ src/mpi/comm/comm_create_group.c | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-179-gfaff4d1
by noreply＠mpich.org 09 May '13

09 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via faff4d1d5c504ca29b4fafe18978ad3c05e0c6df (commit) from e9579e1c6f2ed201225b51901129870e90be734e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/faff4d1d5c504ca29b4fafe18978ad3c0… commit faff4d1d5c504ca29b4fafe18978ad3c05e0c6df Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Thu May 9 14:15:40 2013 -0500 Fix for tt#1832. Typo in mpl's configure which was causing MPL headers to be installed, which should not be needed. No reviewer. diff --git a/src/mpl/configure.ac b/src/mpl/configure.ac index 451ffb4..bc0322c 100644 --- a/src/mpl/configure.ac +++ b/src/mpl/configure.ac @@ -52,7 +52,7 @@ PAC_C_MACRO_VA_ARGS # TODO: could add a --enable-embedded or similar to also control this AC_ARG_VAR([FROM_MPICH],[set to "yes" if building MPL inside of MPICH]) -AM_CONDITIONAL([MPL_EMBEDDED_MODE],[test "x$FROM_MPICH" = "xyes"]) +AM_CONDITIONAL([MPL_EMBEDDED_MODE],[test "x${FROM_MPICH}" = "xyes"]) # support gcov test coverage information PAC_ENABLE_COVERAGE ----------------------------------------------------------------------- Summary of changes: src/mpl/configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-178-ge9579e1
by noreply＠mpich.org 09 May '13

09 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via e9579e1c6f2ed201225b51901129870e90be734e (commit) via 87e84a5e3892a68c503c2059918b31506d9ddc73 (commit) via 5ef1a628b2feb1ce786d95691213171493fecaf9 (commit) via c60642ccede4a913d2b025fd6f65c340fe5a8f5f (commit) via f29783c21fd7c5ea74be0ca3b0a920f48b92c730 (commit) via ef90f1b9309ab8ffc67658b5a4cbf81f3e2a9468 (commit) via 1daeae3155739627d291277c89669f0c633c1e5a (commit) via a45954b594b5a5ba4a54acdcb8fda7c2e819a40f (commit) via dcf62e917496501a74fe6cac918c8967316056dc (commit) via 77999d6f565e65e7f1c5ee9f00dda99b28186f0d (commit) from c478a4fe8d4cad97a2412c13b02d445181f78a56 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/e9579e1c6f2ed201225b51901129870e9… commit e9579e1c6f2ed201225b51901129870e90be734e Author: Charles Archer <archerc(a)us.ibm.com> Date: Fri Apr 5 07:24:24 2013 -0400 use void* for sizeof detection instead of int (ibm) D189618 (ibm) f30c0544fcc37fc18cb0f9b27f414373d5d5308b Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 9a4dfc4..44e0137 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -1344,7 +1344,7 @@ int MPIDI_Banner(char * bufPtr) { } } - if(sizeof(int) == 8) + if(sizeof(void*) == 8) strcpy(type, "64bit (MPI over PAMI)"); else if(sizeof(int) == 4) strcpy(type, "32bit (MPI over PAMI)"); diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c index bf9c423..a41ad77 100644 --- a/src/mpid/pamid/src/mpidi_util.c +++ b/src/mpid/pamid/src/mpidi_util.c @@ -562,7 +562,7 @@ int MPIDI_Print_mpenv(int rank,int size) else strcpy(sender.libpath, "NOT SET"); getcwd(sender.directory, FILENAME_MAX+1); - if(sizeof(int) == 4) + if(sizeof(void*) == 4) sender.mode_64 = 0; else sender.mode_64 = 1; http://git.mpich.org/mpich.git/commitdiff/87e84a5e3892a68c503c2059918b31506… commit 87e84a5e3892a68c503c2059918b31506d9ddc73 Author: Charles Archer <archerc(a)us.ibm.com> Date: Thu Apr 4 11:58:52 2013 -0400 detect 32 bit vs 64 bit via sizeof (ibm) D189618 (ibm) 73870926a4a9598deacbc5c0b91fa6150ef8f7ed Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 7e7d00c..9a4dfc4 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -1344,11 +1344,12 @@ int MPIDI_Banner(char * bufPtr) { } } -#ifndef __32BIT__ - strcpy(type, "64bit (MPI over PAMI)"); -#else - strcpy(type, "32bit (MPI over PAMI)"); -#endif + if(sizeof(int) == 8) + strcpy(type, "64bit (MPI over PAMI)"); + else if(sizeof(int) == 4) + strcpy(type, "32bit (MPI over PAMI)"); + else + strcpy(type, "UNKNOWN-bit (MPI over PAMI)"); sprintf(msgBuf,"MPICH library was compiled on"); diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c index 5f9f2cf..bf9c423 100644 --- a/src/mpid/pamid/src/mpidi_util.c +++ b/src/mpid/pamid/src/mpidi_util.c @@ -561,11 +561,12 @@ int MPIDI_Print_mpenv(int rank,int size) if (cp) strncpy(sender.libpath, cp, MAXPATHLEN-1); else strcpy(sender.libpath, "NOT SET"); getcwd(sender.directory, FILENAME_MAX+1); -#ifdef __32BIT__ - sender.mode_64 = 0; -#else - sender.mode_64 = 1; -#endif + + if(sizeof(int) == 4) + sender.mode_64 = 0; + else + sender.mode_64 = 1; + sender.threaded = 1; /* Always 1 */ cp = getenv("AIXTHREAD_SCOPE"); if (cp) strncpy(sender.thread_scope, cp, 8); http://git.mpich.org/mpich.git/commitdiff/5ef1a628b2feb1ce786d9569121317149… commit 5ef1a628b2feb1ce786d95691213171493fecaf9 Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Tue Mar 26 09:15:47 2013 -0500 Fix alltoall handling of non-PAMI datatypes and non-continuous datatypes. (ibm) 36f5ac777f738068690184193b557fab53b3472f Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c index 646b89f..d970069 100644 --- a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c +++ b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c @@ -45,7 +45,10 @@ int MPIDO_Alltoallv(const void *sendbuf, { TRACE_ERR("Entering MPIDO_Alltoallv\n"); volatile unsigned active = 1; - int sndtypelen, rcvtypelen, snd_contig, rcv_contig; + void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; + void *sbuf = NULL, *rbuf = NULL; + int recvok=PAMI_SUCCESS, sendok=PAMI_SUCCESS; + int sndtypelen, rcvtypelen, snd_contig=0, rcv_contig=0; MPID_Datatype *sdt, *rdt; pami_type_t stype, rtype; MPI_Aint sdt_true_lb, rdt_true_lb; @@ -61,17 +64,20 @@ int MPIDO_Alltoallv(const void *sendbuf, #endif const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); const int selected_type = mpid->user_selected_type[PAMI_XFER_ALLTOALLV_INT]; - - if((sendbuf != MPI_IN_PLACE) && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)) - pamidt = 0; - if(MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS) - pamidt = 0; - - MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvtypelen, rdt, rdt_true_lb); - if(!rcv_contig) pamidt = 0; - - if((selected_type == MPID_COLL_USE_MPICH) || - pamidt == 0) + const int size = comm_ptr->local_size; + int sendcontinuous , recvcontinuous=0; + size_t recv_size=0, send_size=0; + size_t totalrecvcount=0; + int *lrecvdispls = NULL; /* possible local displs calculated for noncontinous */ + int *lsenddispls = NULL;/* possible local displs calculated for noncontinous */ + int *lrecvcounts = NULL;/* possible local counts calculated for noncontinous */ + int *lsendcounts = NULL;/* possible local counts calculated for noncontinous */ + const int *precvdispls = recvdispls; /* pointer to displs to use as pami parmi */ + const int *psenddispls = senddispls; /* pointer to displs to use as pami parmi */ + const int *precvcounts = recvcounts; /* pointer to counts to use as pami parmi */ + const int *psendcounts = sendcounts; /* pointer to counts to use as pami parmi */ + int inplace = sendbuf == MPI_IN_PLACE? 1 : 0; + if(selected_type == MPID_COLL_USE_MPICH) { if(unlikely(verbose)) fprintf(stderr,"Using MPICH alltoallv algorithm\n"); @@ -80,6 +86,148 @@ int MPIDO_Alltoallv(const void *sendbuf, comm_ptr, mpierrno); } + if(!inplace) + { + sendok = MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp); + MPIDI_Datatype_get_info(1, sendtype, snd_contig, sndtypelen, sdt, sdt_true_lb); + sbuf = (char *)sendbuf + sdt_true_lb; + if(!snd_contig || (sendok != PAMI_SUCCESS)) + { + stype = PAMI_TYPE_UNSIGNED_CHAR; + size_t totalsendcount = sendcounts[0]; + sendcontinuous = senddispls[0] == 0? 1 : 0 ; + int i; + psenddispls = lsenddispls = MPIU_Malloc(size*sizeof(int)); + psendcounts = lsendcounts = MPIU_Malloc(size*sizeof(int)); + lsenddispls[0]= 0; + lsendcounts[0]= sndtypelen * sendcounts[0]; + for(i=1; i<size; ++i) + { + lsenddispls[i]= sndtypelen * totalsendcount; + totalsendcount += sendcounts[i]; + if(senddispls[i] != (senddispls[i-1] + (sendcounts[i-1]*sndtypelen))) + sendcontinuous = 0; + lsendcounts[i]= sndtypelen * sendcounts[i]; + } + send_size = sndtypelen * totalsendcount; + TRACE_ERR("Pack receive sndv_contig %zu, sendok %zd, totalsendcount %zu, sendcontinuous %zu, sndtypelen %zu, send_size %zu\n", + (size_t)snd_contig, (size_t)sendok, (size_t)totalsendcount, (size_t)sendcontinuous, (size_t) sndtypelen, (size_t)send_size); + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + if(sendcontinuous) + { + MPIR_Localcopy(sendbuf, totalsendcount, sendtype, + snd_noncontig_buff, send_size,MPI_CHAR); + } + else + { + size_t extent; + MPID_Datatype_get_extent_macro(sendtype,extent); + for(i=0; i<size; ++i) + { + char* scbuf = (char*)sendbuf + senddispls[i]*extent; + char* rcbuf = (char*)snd_noncontig_buff + psenddispls[i]; + MPIR_Localcopy(scbuf, sendcounts[i], sendtype, + rcbuf, psendcounts[i], MPI_CHAR); + TRACE_ERR("Pack send src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n", + (size_t)extent, (size_t)i,(size_t)senddispls[i],(size_t)i,(size_t)sendcounts[i],(size_t)senddispls[i], *(int*)scbuf); + TRACE_ERR("Pack send dest displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n", + (size_t)i,(size_t)psenddispls[i],(size_t)i,(size_t)psendcounts[i],(size_t)psenddispls[i], *(int*)rcbuf); + } + } + } + } + MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvtypelen, rdt, rdt_true_lb); + recvok = MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp); + rbuf = (char *)recvbuf + rdt_true_lb; + if(!rcv_contig || (recvok != PAMI_SUCCESS)) + { + rtype = PAMI_TYPE_UNSIGNED_CHAR; + totalrecvcount = recvcounts[0]; + recvcontinuous = recvdispls[0] == 0? 1 : 0 ; + int i; + precvdispls = lrecvdispls = MPIU_Malloc(size*sizeof(int)); + precvcounts = lrecvcounts = MPIU_Malloc(size*sizeof(int)); + lrecvdispls[0]= 0; + lrecvcounts[0]= rcvtypelen * recvcounts[0]; + for(i=1; i<size; ++i) + { + lrecvdispls[i]= rcvtypelen * totalrecvcount; + totalrecvcount += recvcounts[i]; + if(recvdispls[i] != (recvdispls[i-1] + (recvcounts[i-1]*rcvtypelen))) + recvcontinuous = 0; + lrecvcounts[i]= rcvtypelen * recvcounts[i]; + } + recv_size = rcvtypelen * totalrecvcount; + TRACE_ERR("Pack receive rcv_contig %zu, recvok %zd, totalrecvcount %zu, recvcontinuous %zu, rcvtypelen %zu, recv_size %zu\n", + (size_t)rcv_contig, (size_t)recvok, (size_t)totalrecvcount, (size_t)recvcontinuous,(size_t)rcvtypelen, (size_t)recv_size); + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + /* need to copy it now if it's used for the send buffer and then do not do in place */ + if(inplace) + { + inplace = 0; + stype = PAMI_TYPE_UNSIGNED_CHAR; + size_t totalsendcount = recvcounts[0]; + sendcontinuous = recvdispls[0] == 0? 1 : 0 ; + int i; + psenddispls = lsenddispls = MPIU_Malloc(size*sizeof(int)); + psendcounts = lsendcounts = MPIU_Malloc(size*sizeof(int)); + lsenddispls[0]= 0; + lsendcounts[0]= rcvtypelen * recvcounts[0]; + for(i=1; i<size; ++i) + { + lsenddispls[i]= rcvtypelen * totalsendcount; + totalsendcount += recvcounts[i]; + if(recvdispls[i] != (recvdispls[i-1] + (recvcounts[i-1]*rcvtypelen))) + sendcontinuous = 0; + lsendcounts[i]= rcvtypelen * recvcounts[i]; + } + send_size = rcvtypelen * totalsendcount; + TRACE_ERR("Pack MPI_IN_PLACE receive sndv_contig %zu, sendok %zd, totalsendcount %zu, sendcontinuous %zu, rcvtypelen %zu, send_size %zu\n", + (size_t)snd_contig, (size_t)sendok, (size_t)totalsendcount, (size_t)sendcontinuous, (size_t) rcvtypelen, (size_t)send_size); + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + if(sendcontinuous) + { + MPIR_Localcopy(recvbuf, totalsendcount, recvtype, + snd_noncontig_buff, send_size,MPI_CHAR); + } + else + { + size_t extent; + MPID_Datatype_get_extent_macro(recvtype,extent); + for(i=0; i<size; ++i) + { + char* scbuf = (char*)recvbuf + recvdispls[i]*extent; + char* rcbuf = (char*)snd_noncontig_buff + psenddispls[i]; + MPIR_Localcopy(scbuf, recvcounts[i], recvtype, + rcbuf, psendcounts[i], MPI_CHAR); + TRACE_ERR("Pack send src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n", + (size_t)extent, (size_t)i,(size_t)recvdispls[i],(size_t)i,(size_t)recvcounts[i],(size_t)recvdispls[i], *(int*)scbuf); + TRACE_ERR("Pack send dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n", + (size_t)extent, (size_t)i,(size_t)psenddispls[i],(size_t)i,(size_t)psendcounts[i],(size_t)psenddispls[i], *(int*)rcbuf); + } + } + } + } + + pami_xfer_t alltoallv; pami_algorithm_t my_alltoallv; const pami_metadata_t *my_md = (pami_metadata_t *)NULL; @@ -106,27 +254,26 @@ int MPIDO_Alltoallv(const void *sendbuf, alltoallv.cb_done = cb_alltoallv; alltoallv.cookie = (void *)&active; /* We won't bother with alltoallv since MPI is always going to be ints. */ - if(sendbuf == MPI_IN_PLACE) + if(inplace) { if(unlikely(verbose)) fprintf(stderr,"alltoallv MPI_IN_PLACE buffering\n"); alltoallv.cmd.xfer_alltoallv_int.stype = rtype; - alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) recvdispls; - alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) recvcounts; + alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) precvdispls; + alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) precvcounts; alltoallv.cmd.xfer_alltoallv_int.sndbuf = PAMI_IN_PLACE; } else { - MPIDI_Datatype_get_info(1, sendtype, snd_contig, sndtypelen, sdt, sdt_true_lb); alltoallv.cmd.xfer_alltoallv_int.stype = stype; - alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) senddispls; - alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) sendcounts; - alltoallv.cmd.xfer_alltoallv_int.sndbuf = (char *)sendbuf+sdt_true_lb; + alltoallv.cmd.xfer_alltoallv_int.sdispls = (int *) psenddispls; + alltoallv.cmd.xfer_alltoallv_int.stypecounts = (int *) psendcounts; + alltoallv.cmd.xfer_alltoallv_int.sndbuf = sbuf; } - alltoallv.cmd.xfer_alltoallv_int.rcvbuf = (char *)recvbuf+rdt_true_lb; + alltoallv.cmd.xfer_alltoallv_int.rcvbuf = rbuf; - alltoallv.cmd.xfer_alltoallv_int.rdispls = (int *) recvdispls; - alltoallv.cmd.xfer_alltoallv_int.rtypecounts = (int *) recvcounts; + alltoallv.cmd.xfer_alltoallv_int.rdispls = (int *) precvdispls; + alltoallv.cmd.xfer_alltoallv_int.rtypecounts = (int *) precvcounts; alltoallv.cmd.xfer_alltoallv_int.rtype = rtype; if(unlikely(queryreq == MPID_COLL_ALWAYS_QUERY || @@ -205,6 +352,37 @@ int MPIDO_Alltoallv(const void *sendbuf, TRACE_ERR("%d waiting on active %d\n", rank, active); MPID_PROGRESS_WAIT_WHILE(active); + if(!rcv_contig || (recvok != PAMI_SUCCESS)) + { + if(recvcontinuous) + { + MPIR_Localcopy(rcv_noncontig_buff, recv_size,MPI_CHAR, + recvbuf, totalrecvcount, recvtype); + } + else + { + size_t extent; + int i; + MPID_Datatype_get_extent_macro(recvtype,extent); + for(i=0; i<size; ++i) + { + char* scbuf = (char*)rcv_noncontig_buff+ precvdispls[i]; + char* rcbuf = (char*)recvbuf + recvdispls[i]*extent; + MPIR_Localcopy(scbuf, precvcounts[i], MPI_CHAR, + rcbuf, recvcounts[i], recvtype); + TRACE_ERR("Pack recv src extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n", + (size_t)extent, (size_t)i,(size_t)precvdispls[i],(size_t)i,(size_t)precvcounts[i],(size_t)precvdispls[i], *(int*)scbuf); + TRACE_ERR("Pack recv dest extent %zu, displ[%zu]=%zu, count[%zu]=%zu buf[%zu]=%u\n", + (size_t)extent, (size_t)i,(size_t)recvdispls[i],(size_t)i,(size_t)recvcounts[i],(size_t)recvdispls[i], *(int*)rcbuf); + } + } + MPIU_Free(rcv_noncontig_buff); + } + if(!snd_contig || (sendok != PAMI_SUCCESS)) MPIU_Free(snd_noncontig_buff); + if(lrecvdispls) MPIU_Free(lrecvdispls); + if(lsenddispls) MPIU_Free(lsenddispls); + if(lrecvcounts) MPIU_Free(lrecvcounts); + if(lsendcounts) MPIU_Free(lsendcounts); TRACE_ERR("Leaving alltoallv\n"); http://git.mpich.org/mpich.git/commitdiff/c60642ccede4a913d2b025fd6f65c340f… commit c60642ccede4a913d2b025fd6f65c340fe5a8f5f Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Wed Mar 6 15:41:26 2013 -0600 BGQ must create geometries with (existing) tasklists (ibm) Issue 7967 (ibm) 6b7ba8eb718d8323c8dd638c27cca6a05baa202b Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_macros.h b/src/mpid/pamid/include/mpidi_macros.h index 14d0195..144b0b6 100644 --- a/src/mpid/pamid/include/mpidi_macros.h +++ b/src/mpid/pamid/include/mpidi_macros.h @@ -114,7 +114,14 @@ _dt_contig_out, _data_sz_out, _dt_ptr, _dt_true_lb) \ vcr[index]->taskid; \ }) - +#ifdef __BGQ__ +/* BGQ just shares the MPICH vcr/tasklist. + This relies on the VCR being a simple task list which is asserted + in static_assertions() in mpid_init.c */ +#define MPID_VCR_GET_LPIDS(comm, taskids) taskids = &((*comm->vcr)->taskid); +#define MPID_VCR_FREE_LPIDS(taskids) +#else +/* non-BGQ mallocs and copies the MPICH vcr/tasklist */ #define MPID_VCR_GET_LPIDS(comm, taskids) \ ({ \ int i; \ @@ -123,7 +130,9 @@ _dt_contig_out, _data_sz_out, _dt_ptr, _dt_true_lb) \ for(i=0; i<comm->local_size; i++) \ taskids[i] = comm->vcr[i]->taskid; \ }) +#define MPID_VCR_FREE_LPIDS(taskids) MPIU_Free(taskids) +#endif #define MPID_GPID_Get(comm_ptr, rank, gpid) \ ({ \ diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index 8ee9532..a8be80c 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -245,13 +245,13 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm) comm->mpid.tasks = NULL; for(i=1;i<comm->local_size;i++) { - /* only if sequential tasks should we use a (single) range. - Multi or reordered ranges are inefficient */ - if(MPID_VCR_GET_LPID(comm->vcr, i) != (MPID_VCR_GET_LPID(comm->vcr, i-1) + 1)) { - /* not sequential, use tasklist */ - MPID_VCR_GET_LPIDS(comm, comm->mpid.tasks); - break; - } + /* only if sequential tasks should we use a (single) range. + Multi or reordered ranges are inefficient */ + if(MPID_VCR_GET_LPID(comm->vcr, i) != (MPID_VCR_GET_LPID(comm->vcr, i-1) + 1)) { + /* not sequential, use tasklist */ + MPID_VCR_GET_LPIDS(comm, comm->mpid.tasks); + break; + } } /* Should we use a range? (no task list set) */ if(comm->mpid.tasks == NULL) @@ -421,7 +421,7 @@ void MPIDI_Coll_comm_destroy(MPID_Comm *comm) TRACE_ERR("Waiting for geom destroy to finish\n"); MPID_PROGRESS_WAIT_WHILE(geom_destroy); - MPIU_Free(comm->mpid.tasks); + MPID_VCR_FREE_LPIDS(comm->mpid.tasks); /* TRACE_ERR("Freeing geometry ranges\n"); MPIU_TestFree(&comm->mpid.tasks_descriptor.ranges); */ diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 12983fc..7e7d00c 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -1374,6 +1374,7 @@ int MPIDI_Banner(char * bufPtr) { sprintf(bufPtr, "%s %s %s %s ", type, ver_buf, msgBuf, ASC_time); } else { sprintf(bufPtr, "%s %s %s ", type, msgBuf, ASC_time); + } MPIU_Free(tmx); @@ -1388,6 +1389,11 @@ static_assertions() MPID_assert_static(sizeof(void*) == sizeof(size_t)); MPID_assert_static(sizeof(uintptr_t) == sizeof(size_t)); #ifdef __BGQ__ + /* MPID_VCR_GET_LPIDS relies on the VCR being a simple task list */ + MPID_VCR vcr=NULL; + MPID_assert_static(sizeof(*vcr) == sizeof(pami_task_t));/* VCR is a simple task list */ + MPID_assert_static(sizeof(vcr->taskid) == sizeof(*vcr));/* VCR is a simple task list */ + MPID_assert_static(sizeof(MPIDI_MsgInfo) == 16); MPID_assert_static(sizeof(uint64_t) == sizeof(size_t)); #endif http://git.mpich.org/mpich.git/commitdiff/f29783c21fd7c5ea74be0ca3b0a920f48… commit f29783c21fd7c5ea74be0ca3b0a920f48b92c730 Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Thu Mar 21 11:58:55 2013 -0500 Remove ticket #595 circumvention (ibm) Trac #659 (ibm) 9d36aed8f988859eba1a284292f0a1a6fc75d01d Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/comm/mpid_selectcolls.c b/src/mpid/pamid/src/comm/mpid_selectcolls.c index cd30053..571eb68 100644 --- a/src/mpid/pamid/src/comm/mpid_selectcolls.c +++ b/src/mpid/pamid/src/comm/mpid_selectcolls.c @@ -370,15 +370,6 @@ void MPIDI_Comm_coll_envvars(MPID_Comm *comm) { /* In addition to glue protocols, check for other PAMI protocols and check for PE now */ char* names[] = {"PAMID_COLLECTIVE_SCATTERV", NULL}; MPIDI_Check_protocols(names, comm, "scatterv", PAMI_XFER_SCATTERV_INT); - - /* Use MPICH on large communicators (Issue 7516 and ticket 595)*/ - if((comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == - MPID_COLL_NOSELECTION) /* no env var selected */ - && (comm->local_size > (16*1024))) /* and > 16k ranks */ - { - comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_USE_MPICH; - comm->mpid.user_selected[PAMI_XFER_SCATTERV_INT] = 0; - } } TRACE_ERR("Checking scatter\n"); http://git.mpich.org/mpich.git/commitdiff/ef90f1b9309ab8ffc67658b5a4cbf81f3… commit ef90f1b9309ab8ffc67658b5a4cbf81f3e2a9468 Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Mon Mar 11 18:17:03 2013 -0500 Update scatter[v] protocol selection Do not tie scatterv to scatter selection. (ibm) Issue 9416 (ibm) 4901e679071e489e38ee1a6925da36eab9c3ffac Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index 70a25ee..8ee9532 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -237,7 +237,7 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm) if(comm->mpid.geometry != MPIDI_Process.world_geometry) { - if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL) + if(unlikely(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)) fprintf(stderr,"world geom: %p parent geom: %p\n", MPIDI_Process.world_geometry, comm->mpid.parent); TRACE_ERR("Creating subgeom\n"); /* Change to this at some point */ diff --git a/src/mpid/pamid/src/comm/mpid_optcolls.c b/src/mpid/pamid/src/comm/mpid_optcolls.c index 23f63d7..03192f3 100644 --- a/src/mpid/pamid/src/comm/mpid_optcolls.c +++ b/src/mpid/pamid/src/comm/mpid_optcolls.c @@ -273,6 +273,128 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr) opt_proto = -1; mustquery = 0; + /* Scatter */ + /* If the user has forced a selection, don't bother setting it here */ + if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_NOSELECTION) + { + TRACE_ERR("No scatter env var, so setting optimized scatter\n"); + /* The best scatter seems to be I0:Binomial (at least better than I0:Flat) */ + /* Loop over the protocols until we find the one we want */ + for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTER][0]; i++) + { + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][0][i].name, "I0:Binomial:-:MU") == 0) + { + opt_proto = i; + break; + } + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][0][i].name, "I0:Binomial:SHMEM:MU") == 0) + { + opt_proto = i; + break; + } + } + if(opt_proto == -1) /* check other list */ + for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTER][1]; i++) + { + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][1][i].name, "I0:Binomial:-:MU") == 0) + { + opt_proto = i; + mustquery = 1; + break; + } + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][1][i].name, "I0:Binomial:SHMEM:MU") == 0) + { + opt_proto = i; + mustquery = 1; + break; + } + } + /* Actually, MPICH looks better right now, so use MPICH + if(opt_proto != -1) + { + TRACE_ERR("Memcpy protocol type %d, number %d (%s) to optimized protocol\n", + PAMI_XFER_SCATTER, opt_proto, + comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][mustquery][opt_proto].name); + comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTER][0] = + comm_ptr->mpid.coll_algorithm[PAMI_XFER_SCATTER][mustquery][opt_proto]; + memcpy(&comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTER][0], + &comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTER][mustquery][opt_proto], + sizeof(pami_metadata_t)); + comm_ptr->mpid.must_query[PAMI_XFER_SCATTER][0] = mustquery?MPID_COLL_ALWAYS_QUERY:MPID_COLL_NOQUERY; + comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] = MPID_COLL_OPTIMIZED; + } + else + */ + { + TRACE_ERR("Scatter reverting to MPICH\n"); + comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] = MPID_COLL_USE_MPICH; + comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTER][0] = 0; + } + TRACE_ERR("Done setting optimized scatter\n"); + } + opt_proto = -1; + mustquery = 0; + /* Scatterv */ + /* If the user has forced a selection, don't bother setting it here */ + if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_NOSELECTION) + { + TRACE_ERR("No scatterv env var, so setting optimized scatterv\n"); + /* The best scatterv seems to be I0:SoftwareTree: */ + /* Loop over the protocols until we find the one we want */ + for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTERV_INT][0]; i++) + { + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][0][i].name, "I0:SoftwareTree:-:MU") == 0) + { + opt_proto = i; + break; + } + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][0][i].name, "I0:SoftwareTree:SHMEM:MU") == 0) + { + opt_proto = i; + break; + } + } + if(opt_proto == -1) /* check other list */ + for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_SCATTERV_INT][1]; i++) + { + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][1][i].name, "I0:SoftwareTree:-:MU") == 0) + { + opt_proto = i; + mustquery = 1; + break; + } + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][1][i].name, "I0:SoftwareTree:SHMEM:MU") == 0) + { + opt_proto = i; + mustquery = 1; + break; + } + } + if(opt_proto != -1) + { + TRACE_ERR("Memcpy protocol type %d, number %d (%s) to optimized protocol\n", + PAMI_XFER_SCATTERV_INT, opt_proto, + comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][mustquery][opt_proto].name); + comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTERV_INT][0] = + comm_ptr->mpid.coll_algorithm[PAMI_XFER_SCATTERV_INT][mustquery][opt_proto]; + memcpy(&comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTERV_INT][0], + &comm_ptr->mpid.coll_metadata[PAMI_XFER_SCATTERV_INT][mustquery][opt_proto], + sizeof(pami_metadata_t)); + comm_ptr->mpid.must_query[PAMI_XFER_SCATTERV_INT][0] = mustquery?MPID_COLL_ALWAYS_QUERY:MPID_COLL_NOQUERY; + comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_OPTIMIZED; + } + else + { + TRACE_ERR("Scatterv reverting to MPICH\n"); + comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_USE_MPICH; + comm_ptr->mpid.opt_protocol[PAMI_XFER_SCATTERV_INT][0] = 0; + } + TRACE_ERR("Done setting optimized scatterv\n"); + } + + + opt_proto = -1; + mustquery = 0; /* Alltoall */ /* If the user has forced a selection, don't bother setting it here */ if(comm_ptr->mpid.user_selected_type[PAMI_XFER_ALLTOALL] == MPID_COLL_NOSELECTION) @@ -859,6 +981,18 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr) fprintf(stderr,"Selecting MPICH for allgatherv below %d size comm %p\n", comm_ptr->mpid.cutoff_size[PAMI_XFER_ALLGATHERV_INT][0], comm_ptr); if(comm_ptr->mpid.user_selected_type[PAMI_XFER_GATHER] == MPID_COLL_USE_MPICH) fprintf(stderr,"Selecting MPICH for gather comm %p\n", comm_ptr); + if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_USE_MPICH) + fprintf(stderr,"Selecting MPICH for scatter comm %p\n", comm_ptr); + else if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_OPTIMIZED) + fprintf(stderr,"Selecting %s for scatter comm %p\n", + comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTER][0].name, + comm_ptr); + if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_USE_MPICH) + fprintf(stderr,"Selecting MPICH for scatterv comm %p\n", comm_ptr); + else if(comm_ptr->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_OPTIMIZED) + fprintf(stderr,"Selecting %s for scatterv comm %p\n", + comm_ptr->mpid.opt_protocol_md[PAMI_XFER_SCATTERV_INT][0].name, + comm_ptr); if(comm_ptr->mpid.user_selected_type[PAMI_XFER_BROADCAST] == MPID_COLL_OPTIMIZED) fprintf(stderr,"Selecting %s for opt bcast up to size %d comm %p\n", comm_ptr->mpid.opt_protocol_md[PAMI_XFER_BROADCAST][0].name, comm_ptr->mpid.cutoff_size[PAMI_XFER_BROADCAST][0], comm_ptr); http://git.mpich.org/mpich.git/commitdiff/1daeae3155739627d291277c89669f0c6… commit 1daeae3155739627d291277c89669f0c633c1e5a Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Mon Mar 11 17:24:14 2013 -0500 Fix M2M protocol selection (ibm) Issue 9416 (ibm) 723fb4ce39b5e06ef63cc37231acdaeb6d1c08cf Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/comm/mpid_optcolls.c b/src/mpid/pamid/src/comm/mpid_optcolls.c index 85bfd45..23f63d7 100644 --- a/src/mpid/pamid/src/comm/mpid_optcolls.c +++ b/src/mpid/pamid/src/comm/mpid_optcolls.c @@ -278,10 +278,18 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr) if(comm_ptr->mpid.user_selected_type[PAMI_XFER_ALLTOALL] == MPID_COLL_NOSELECTION) { TRACE_ERR("No alltoall env var, so setting optimized alltoall\n"); - /* the best alltoall is always I0:M2MComposite:MU:MU, though there are - * displacement array memory issues today.... */ + /* The best alltoall is always I0:M2MComposite:MU:MU */ /* Loop over the protocols until we find the one we want */ if(use_threaded_collectives) + for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALL][0]; i++) + { + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALL][0][i].name, "I0:M2MComposite:MU:MU") == 0) + { + opt_proto = i; + break; + } + } + if(use_threaded_collectives && (opt_proto == -1)) /* check other list */ for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALL][1]; i++) { if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALL][1][i].name, "I0:M2MComposite:MU:MU") == 0) @@ -290,7 +298,7 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr) mustquery = 1; break; } - } + } if(opt_proto != -1) { TRACE_ERR("Memcpy protocol type %d, number %d (%s) to optimized protocol\n", @@ -320,10 +328,18 @@ void MPIDI_Comm_coll_select(MPID_Comm *comm_ptr) if(comm_ptr->mpid.user_selected_type[PAMI_XFER_ALLTOALLV_INT] == MPID_COLL_NOSELECTION) { TRACE_ERR("No alltoallv env var, so setting optimized alltoallv\n"); - /* the best alltoallv is always I0:M2MComposite:MU:MU, though there are - * displacement array memory issues today.... */ + /* The best alltoall is always I0:M2MComposite:MU:MU */ /* Loop over the protocols until we find the one we want */ if(use_threaded_collectives) + for(i = 0; i < comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALLV_INT][0]; i++) + { + if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALLV_INT][0][i].name, "I0:M2MComposite:MU:MU") == 0) + { + opt_proto = i; + break; + } + } + if(use_threaded_collectives && (opt_proto == -1)) /* check other list */ for(i = 0; i <comm_ptr->mpid.coll_count[PAMI_XFER_ALLTOALLV_INT][1]; i++) { if(strcasecmp(comm_ptr->mpid.coll_metadata[PAMI_XFER_ALLTOALLV_INT][1][i].name, "I0:M2MComposite:MU:MU") == 0) http://git.mpich.org/mpich.git/commitdiff/a45954b594b5a5ba4a54acdcb8fda7c2e… commit a45954b594b5a5ba4a54acdcb8fda7c2e819a40f Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Mon Mar 11 12:16:38 2013 -0500 Need to read some environment variables before creating the client (ibm) 26edfa903df4881e1ab0190bc1859f5dc30e868e Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 2f5394f..12983fc 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -294,6 +294,14 @@ MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char ** pami_result_t rc = PAMI_ERROR; pami_configuration_t config[2]; + + /* Set the status for memory optimized collectives */ + { + char* env = getenv("PAMID_COLLECTIVES_MEMORY_OPTIMIZED"); + if (env != NULL) + MPIDI_atoi(env,&MPIDI_Process.optimized.memory); + } + config[0].name = PAMI_CLIENT_NONCONTIG; if(MPIDI_Process.optimized.memory & MPID_OPT_LVL_NONCONTIG) config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives so save memory http://git.mpich.org/mpich.git/commitdiff/dcf62e917496501a74fe6cac918c89673… commit dcf62e917496501a74fe6cac918c8967316056dc Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Thu Mar 7 14:21:02 2013 -0600 Use MPIDO and not MPIR internally in PAMID (ibm) 88338876fc003a9fe936298097b42df1e54ead32 Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/coll/gather/mpido_gather.c b/src/mpid/pamid/src/coll/gather/mpido_gather.c index 1047a44..8829c2b 100644 --- a/src/mpid/pamid/src/coll/gather/mpido_gather.c +++ b/src/mpid/pamid/src/coll/gather/mpido_gather.c @@ -102,8 +102,8 @@ int MPIDO_Gather_reduce(void * sendbuf, memset(tempbuf, 0, sbytes * size * sizeof(char)); memcpy(tempbuf+(rank*sbytes), sendbuf, sbytes); } - /* #warning TODO need an optimal reduce */ - rc = MPIR_Reduce(MPI_IN_PLACE, + /* Switch to comm->coll_fns->fn() */ + rc = MPIDO_Reduce(MPI_IN_PLACE, tempbuf, (sbytes * size)/4, MPI_INT, diff --git a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c index 00946a3..bc47e41 100644 --- a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c +++ b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c @@ -175,7 +175,8 @@ int MPIDO_Scatterv_alltoallv(void * sendbuf, rcounts[root] = rbytes; - rc = MPIR_Alltoallv(sbuf, + /* Switch to comm->coll_fns->fn() */ + rc = MPIDO_Alltoallv(sbuf, scounts, sdispls, sendtype, diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index d3b03c9..70a25ee 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -353,7 +353,8 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm) MPIDI_Comm_coll_select(comm); TRACE_ERR("mpir barrier\n"); int mpierrno; - MPIR_Barrier(comm, &mpierrno); + /* Switch to comm->coll_fns->fn() */ + MPIDO_Barrier(comm, &mpierrno); TRACE_ERR("MPIDI_Coll_comm_create exit\n"); http://git.mpich.org/mpich.git/commitdiff/77999d6f565e65e7f1c5ee9f00dda99b2… commit 77999d6f565e65e7f1c5ee9f00dda99b28186f0d Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Mon Mar 18 14:51:05 2013 -0500 Specify PAMID_COLLECTIVES_MEMORY_OPTIMIZED levels (ibm) CPS 95PHUZ (ibm) 1b4e2da6d5fb0990d530be6f896f539dc7571620 Signed-off-by: sssharka <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_constants.h b/src/mpid/pamid/include/mpidi_constants.h index a16292e..42d4bd8 100644 --- a/src/mpid/pamid/include/mpidi_constants.h +++ b/src/mpid/pamid/include/mpidi_constants.h @@ -117,4 +117,10 @@ enum MPID_AUTO_SELECT_COLLS_ALL = 0xFFFFFFFF, }; +enum /* PAMID_COLLECTIVES_MEMORY_OPTIMIZED levels */ + +{ + MPID_OPT_LVL_IRREG = 1, /**< Do not optimize irregular communicators */ + MPID_OPT_LVL_NONCONTIG = 2, /**< Disable some non-contig collectives */ +}; #endif diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 42f0da0..7e33c76 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -113,7 +113,7 @@ typedef struct unsigned subcomms; /**< Enable hardware optimized subcomm's */ unsigned select_colls; /**< Enable collective selection */ unsigned auto_select_colls; /**< Enable automatic collective selection */ - unsigned memory; /**< Enable memory optimized subcomm's */ + unsigned memory; /**< Enable memory optimized subcomm's - See MPID_OPT_LVL_xxxx */ unsigned num_requests; /**< Number of requests between flow control barriers */ } optimized; diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index 3201595..d3b03c9 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -266,7 +266,11 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm) pami_configuration_t config[3]; config[0].name = PAMI_GEOMETRY_NONCONTIG; - config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives + if(MPIDI_Process.optimized.memory & MPID_OPT_LVL_NONCONTIG) + config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives + else + config[0].value.intval = 1; // Enable non-contig even though pamid doesn't use pami for non-contig data collectives, + // we still possibly want those collectives for other reasons. size_t numconfigs = 1; if(MPIDI_Process.optimized.subcomms) { @@ -281,7 +285,7 @@ void MPIDI_Coll_comm_create(MPID_Comm *comm) ++numconfigs; } - if(MPIDI_Process.optimized.memory && (comm->local_size & (comm->local_size-1))) + if((MPIDI_Process.optimized.memory & MPID_OPT_LVL_IRREG) && (comm->local_size & (comm->local_size-1))) { /* Don't create irregular geometries. Fallback to MPICH only collectives */ geom_init = 0; diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 4f85bd6..2f5394f 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -295,7 +295,11 @@ MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char ** pami_configuration_t config[2]; config[0].name = PAMI_CLIENT_NONCONTIG; - config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data + if(MPIDI_Process.optimized.memory & MPID_OPT_LVL_NONCONTIG) + config[0].value.intval = 0; // Disable non-contig, pamid doesn't use pami for non-contig data collectives so save memory + else + config[0].value.intval = 1; // Enable non-contig even though pamid doesn't use pami for non-contig data collectives, + // we still possibly want those collectives for other reasons. size_t numconfigs = 1; if(MPIDI_Process.optimized.memory) { diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c index f08acb7..f32fccd 100644 --- a/src/mpid/pamid/src/mpidi_env.c +++ b/src/mpid/pamid/src/mpidi_env.c @@ -110,8 +110,9 @@ * optimized to reduce memory usage. This may disable some PAMI collectives. * Possible values: * - 0 - Collectives are not memory optimized. - * - n - Collectives are memory optimized. 'n' may represent different - * levels of optimization. + * - n - Collectives are memory optimized. Levels are bitwise values : + * MPID_OPT_LVL_IRREG = 1, Do not optimize irregular communicators + * MPID_OPT_LVL_NONCONTIG = 2, Disable some non-contig collectives * * PAMID_OPTIMIZED_SUBCOMMS - Use PAMI 'optimized' collectives. Defaullt is 1. * - 0 - Some optimized protocols may be disabled. ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_constants.h | 6 + src/mpid/pamid/include/mpidi_datatypes.h | 2 +- src/mpid/pamid/include/mpidi_macros.h | 11 +- .../pamid/src/coll/alltoallv/mpido_alltoallv.c | 222 ++++++++++++++++++-- src/mpid/pamid/src/coll/gather/mpido_gather.c | 4 +- src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c | 3 +- src/mpid/pamid/src/comm/mpid_comm.c | 29 ++- src/mpid/pamid/src/comm/mpid_optcolls.c | 160 ++++++++++++++- src/mpid/pamid/src/comm/mpid_selectcolls.c | 9 - src/mpid/pamid/src/mpid_init.c | 31 +++- src/mpid/pamid/src/mpidi_env.c | 5 +- src/mpid/pamid/src/mpidi_util.c | 11 +- 12 files changed, 427 insertions(+), 66 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-168-gc478a4f
by noreply＠mpich.org 08 May '13

08 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via c478a4fe8d4cad97a2412c13b02d445181f78a56 (commit) via 50887ce11493c5aa872e3257b2e406174b59a12b (commit) via 5d6aefa53ac174f8e1c4e3fbbd5fdf0225f48108 (commit) from 5c4f5de7563db2bee452a102614c53b8ed9583ed (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/c478a4fe8d4cad97a2412c13b02d44518… commit c478a4fe8d4cad97a2412c13b02d445181f78a56 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Thu May 2 22:46:41 2013 -0500 Flag push/pop cleanup. PAC_VAR_PUSHFLAG and PAC_VAR_POPFLAG are duplicates of PAC_PUSH_FLAG and PAC_POP_FLAG. Also, unlike the PAC_VAR_ macro variants, the flag is not updated by the PAC_PUSH/PAC_POP macros, so the older value is retained. Also cleaned up macros that do additional logic to replace empty action-if-true/action-if-false statements with ":". This is not required. The macros work fine without this replacement. Reviewed by gropp. diff --git a/confdb/aclocal_runlog.m4 b/confdb/aclocal_runlog.m4 index 2fdbfa5..e7b5b32 100644 --- a/confdb/aclocal_runlog.m4 +++ b/confdb/aclocal_runlog.m4 @@ -44,82 +44,6 @@ PAC_COMMAND_IFELSE([$1 > $pac_TESTLOG],[ rm -f $pac_TESTLOG ]) - -dnl PAC_VAR_PUSHVAL(VARNAME, [LastSavedValue])) -dnl -dnl Save the content of the shell variable, VARNAME, onto a stack. -dnl The saved value of VARNAME is restorable with respect to the nesting -dnl of the macro. -dnl -dnl The Last saved value of VARNAME on the stack is stored in shell variable -dnl pac_LastSavedValueOf_$VARNAME if the 2nd argument is NOT supplied. -dnl If the 2nd argument is present, the last saved value will be stored -dnl in the 2nd argument instead. -dnl -dnl The First saved value of VARNAME on the stack is stored in shell variable -dnl dnl pac_FirstSavedValueOf_$VARNAME. -dnl -AC_DEFUN([PAC_VAR_PUSHVAL],[ -# START of PUSHVAL -dnl define local m4-name pac_stk_level. -AS_VAR_PUSHDEF([pac_stk_level], [pac_stk_$1_level]) -AS_VAR_SET_IF([pac_stk_level],[ - dnl autoconf < 2.64 does not have AS_VAR_ARITH, so use expr instead. - AS_VAR_SET([pac_stk_level], [`expr $pac_stk_level + 1`]) -],[ - AS_VAR_SET([pac_stk_level], [0]) -]) -dnl AS_ECHO_N(["PUSHVAL: pac_stk_level = $pac_stk_level, "]) -dnl Save the content of VARNAME, i.e. $VARNAME, onto the stack. -AS_VAR_SET([pac_stk_$1_$pac_stk_level],[$$1]) -AS_VAR_IF([pac_stk_level], [0], [ - dnl Save the 1st pushed value of VARNAME as pac_FirstSavedValueOf_$VARNAME - AS_VAR_COPY([pac_FirstSavedValueOf_$1],[pac_stk_$1_$pac_stk_level]) -]) -ifelse([$2],[],[ - dnl Save the last pushed value of VARNAME as pac_LastSavedValueOf_$VARNAME - AS_VAR_COPY([pac_LastSavedValueOf_$1],[pac_stk_$1_$pac_stk_level]) - dnl AS_ECHO(["pac_LastSavedValueOf_$1 = $pac_LastSavedValueOf_$1"]) -],[ - dnl Save the last pushed value of VARNAME as $2 - AS_VAR_COPY([$2],[pac_stk_$1_$pac_stk_level]) - dnl AS_ECHO(["$2 = $$2"]) -]) -AS_VAR_POPDEF([pac_stk_level]) -# END of PUSHVAL -]) -dnl -dnl -dnl -dnl PAC_VAR_POPVAL(VARNAME) -dnl -dnl Restore variable, VARNAME, from the stack. -dnl This macro is safe with respect to the nesting. -dnl Some minimal checking of nesting balance of PAC_VAR_PUSH[POP]VAL() -dnl is done here. -dnl -AC_DEFUN([PAC_VAR_POPVAL],[ -# START of POPVAL -dnl define local m4-name pac_stk_level. -AS_VAR_PUSHDEF([pac_stk_level], [pac_stk_$1_level]) -AS_VAR_SET_IF([pac_stk_level],[ - AS_VAR_IF([pac_stk_level],[-1],[ - AC_MSG_WARN(["Imbalance of PUSHVAL/POPVAL of $1"]) - ],[ - dnl AS_ECHO_N(["POPVAL: pac_stk_level = $pac_stk_level, "]) - AS_VAR_COPY([$1],[pac_stk_$1_$pac_stk_level]) - dnl AS_ECHO(["popped_val = $$1"]) - dnl autoconf < 2.64 does not have AS_VAR_ARITH, so use expr instead. - AS_VAR_SET([pac_stk_level], [`expr $pac_stk_level - 1`]) - ]) -],[ - AC_MSG_WARN(["Uninitialized PUSHVAL/POPVAL of $1"]) -]) -AS_VAR_POPDEF([pac_stk_level]) -# END of POPVAL -]) -dnl -dnl dnl dnl PAC_COMPILE_IFELSE_LOG is a wrapper around AC_COMPILE_IFELSE with the dnl output of ac_compile to a specified logfile instead of AS_MESSAGE_LOG_FD @@ -143,20 +67,12 @@ dnl dnl Replace ">&AS_MESSAGE_LOG_FD" by "> FILE 2>&1" in ac_compile. dnl Save a copy of ac_compile on a stack dnl which is safe through nested invocations of this macro. -PAC_VAR_PUSHVAL([ac_compile]) -dnl Modify ac_compile based on the unmodified ac_compile. -ac_compile="`echo $pac_FirstSavedValueOf_ac_compile | sed -e 's|>.*$|> $1 2>\&1|g'`" -AC_COMPILE_IFELSE([$2],[ - ifelse([$3],[],[:],[$3]) -],[ - ifelse([$4],[],[:],[$4]) +PAC_PUSH_FLAG([ac_compile]) +ac_compile="`echo $ac_compile | sed -e 's|>.*$|> $1 2>\&1|g'`" +AC_COMPILE_IFELSE([$2],[$3],[$4]) +PAC_POP_FLAG([ac_compile]) ]) -dnl Restore the original ac_compile from the stack. -PAC_VAR_POPVAL([ac_compile]) -]) -dnl -dnl -dnl + dnl PAC_LINK_IFELSE_LOG is a wrapper around AC_LINK_IFELSE with the dnl output of ac_link to a specified logfile instead of AS_MESSAGE_LOG_FD dnl @@ -179,20 +95,12 @@ dnl dnl Replace ">&AS_MESSAGE_LOG_FD" by "> FILE 2>&1" in ac_link. dnl Save a copy of ac_link on a stack dnl which is safe through nested invocations of this macro. -PAC_VAR_PUSHVAL([ac_link]) -dnl Modify ac_link based on the unmodified ac_link. -ac_link="`echo $pac_FirstSavedValueOf_ac_link | sed -e 's|>.*$|> $1 2>\&1|g'`" -dnl -AC_LINK_IFELSE([$2],[ - ifelse([$3],[],[:],[$3]) -],[ - ifelse([$4],[],[:],[$4]) -]) -dnl Restore the original ac_link from the stack. -PAC_VAR_POPVAL([ac_link]) +PAC_PUSH_FLAG([ac_link]) +ac_link="`echo $ac_link | sed -e 's|>.*$|> $1 2>\&1|g'`" +AC_LINK_IFELSE([$2],[$3],[$4]) +PAC_POP_FLAG([ac_link]) ]) -dnl -dnl + dnl dnl PAC_COMPLINK_IFELSE (input1, input2, [action-if-true], [action-if-false]) dnl @@ -205,16 +113,10 @@ dnl AC_DEFUN([PAC_COMPLINK_IFELSE],[ AC_COMPILE_IFELSE([$1],[ PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT]) - PAC_VAR_PUSHVAL([LIBS]) - LIBS="pac_conftest.$OBJEXT $pac_FirstSavedValueOf_LIBS" - AC_LINK_IFELSE([$2],[ - ifelse([$3],[],[:],[$3]) - ],[ - ifelse([$4],[],[:],[$4]) - ]) - PAC_VAR_POPVAL([LIBS]) + PAC_PUSH_FLAG([LIBS]) + LIBS="pac_conftest.$OBJEXT $LIBS" + AC_LINK_IFELSE([$2],[$3],[$4]) + PAC_POP_FLAG([LIBS]) rm -f pac_conftest.$OBJEXT -],[ - ifelse([$4],[],[:],[$4]) -]) +],[$4]) ]) http://git.mpich.org/mpich.git/commitdiff/50887ce11493c5aa872e3257b2e406174… commit 50887ce11493c5aa872e3257b2e406174b59a12b Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Thu May 2 22:39:50 2013 -0500 Get rid of PAC_EVAL_IFELSE which is a duplicate of PAC_COMMAND_IFELSE. Reviewed by gropp. diff --git a/confdb/aclocal_runlog.m4 b/confdb/aclocal_runlog.m4 index d78c3e6..2fdbfa5 100644 --- a/confdb/aclocal_runlog.m4 +++ b/confdb/aclocal_runlog.m4 @@ -9,6 +9,7 @@ AC_DEFUN([PAC_RUNLOG],[ ac_status=$? AS_ECHO(["$as_me:$LINENO: \$? = $ac_status"]) >&AS_MESSAGE_LOG_FD test $ac_status = 0; }]) + dnl dnl PAC_COMMAND_IFELSE is written to replace AC_TRY_EVAL with added logging dnl to config.log, i.e. AC_TRY_EVAL does not log anything to config.log. @@ -18,8 +19,6 @@ dnl dnl PAC_COMMAND_IFELSE(COMMMAND,[ACTION-IF-RUN-OK],[ACTION-IF-RUN-FAIL]) dnl AC_DEFUN([PAC_COMMAND_IFELSE],[ -dnl Should use _AC_DO_TOKENS but use AC_RUN_LOG instead -dnl because _AC_XX is autoconf's undocumented macro. AS_IF([PAC_RUNLOG([$1])],[ $2 ],[ @@ -30,25 +29,7 @@ AS_IF([PAC_RUNLOG([$1])],[ ]) ]) ]) -dnl -dnl -dnl -AC_DEFUN([PAC_EVAL_IFELSE],[ -dnl Should use _AC_DO_TOKENS but use AC_RUN_LOG instead -dnl because _AC_XX is autoconf's undocumented macro. -AS_IF([PAC_RUNLOG([$$1])],[ - $2 -],[ - AS_ECHO(["$as_me: program exited with status $ac_status"]) >&AS_MESSAGE_LOG_FD - m4_ifvaln([$3],[ - (exit $ac_status) - $3 - ]) -]) -]) -dnl -dnl -dnl + AC_DEFUN([PAC_RUNLOG_IFELSE],[ dnl pac_TESTLOG is the internal temporary logfile for this macro. pac_TESTLOG="pac_test.log" http://git.mpich.org/mpich.git/commitdiff/5d6aefa53ac174f8e1c4e3fbbd5fdf022… commit 5d6aefa53ac174f8e1c4e3fbbd5fdf0225f48108 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Thu May 2 22:34:58 2013 -0500 PAS_VAR_COPY cleanup. We no longer need PAS_VAR_COPY since AS_VAR_COPY is present in newer autoconf versions. Reviewed by gropp. diff --git a/confdb/aclocal_runlog.m4 b/confdb/aclocal_runlog.m4 index 9b85db5..d78c3e6 100644 --- a/confdb/aclocal_runlog.m4 +++ b/confdb/aclocal_runlog.m4 @@ -62,21 +62,8 @@ PAC_COMMAND_IFELSE([$1 > $pac_TESTLOG],[ ]) rm -f $pac_TESTLOG ]) -dnl -dnl -dnl -dnl PAS_VAR_COPY - A portable layer that mimics AS_VAR_COPY when it is not -dnl defined as in older autoconf, e.g. 2.63 and older. -dnl This macro is absolutely necessary, because AS_VAR_GET in -dnl some newer autoconf, e.g. 2.64, seems to be totally broken, -dnl or behave very different from older autoconf, i.e. 2.63. -dnl -AC_DEFUN([PAS_VAR_COPY],[ -m4_ifdef([AS_VAR_COPY], [AS_VAR_COPY([$1],[$2])], [$1=AS_VAR_GET([$2])]) -]) -dnl -dnl -dnl + + dnl PAC_VAR_PUSHVAL(VARNAME, [LastSavedValue])) dnl dnl Save the content of the shell variable, VARNAME, onto a stack. @@ -106,15 +93,15 @@ dnl Save the content of VARNAME, i.e. $VARNAME, onto the stack. AS_VAR_SET([pac_stk_$1_$pac_stk_level],[$$1]) AS_VAR_IF([pac_stk_level], [0], [ dnl Save the 1st pushed value of VARNAME as pac_FirstSavedValueOf_$VARNAME - PAS_VAR_COPY([pac_FirstSavedValueOf_$1],[pac_stk_$1_$pac_stk_level]) + AS_VAR_COPY([pac_FirstSavedValueOf_$1],[pac_stk_$1_$pac_stk_level]) ]) ifelse([$2],[],[ dnl Save the last pushed value of VARNAME as pac_LastSavedValueOf_$VARNAME - PAS_VAR_COPY([pac_LastSavedValueOf_$1],[pac_stk_$1_$pac_stk_level]) + AS_VAR_COPY([pac_LastSavedValueOf_$1],[pac_stk_$1_$pac_stk_level]) dnl AS_ECHO(["pac_LastSavedValueOf_$1 = $pac_LastSavedValueOf_$1"]) ],[ dnl Save the last pushed value of VARNAME as $2 - PAS_VAR_COPY([$2],[pac_stk_$1_$pac_stk_level]) + AS_VAR_COPY([$2],[pac_stk_$1_$pac_stk_level]) dnl AS_ECHO(["$2 = $$2"]) ]) AS_VAR_POPDEF([pac_stk_level]) @@ -139,7 +126,7 @@ AS_VAR_SET_IF([pac_stk_level],[ AC_MSG_WARN(["Imbalance of PUSHVAL/POPVAL of $1"]) ],[ dnl AS_ECHO_N(["POPVAL: pac_stk_level = $pac_stk_level, "]) - PAS_VAR_COPY([$1],[pac_stk_$1_$pac_stk_level]) + AS_VAR_COPY([$1],[pac_stk_$1_$pac_stk_level]) dnl AS_ECHO(["popped_val = $$1"]) dnl autoconf < 2.64 does not have AS_VAR_ARITH, so use expr instead. AS_VAR_SET([pac_stk_level], [`expr $pac_stk_level - 1`]) ----------------------------------------------------------------------- Summary of changes: confdb/aclocal_runlog.m4 | 166 +++++----------------------------------------- 1 files changed, 18 insertions(+), 148 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-165-g5c4f5de
by noreply＠mpich.org 08 May '13

08 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 5c4f5de7563db2bee452a102614c53b8ed9583ed (commit) from 07de139fa7b3cba2894ddf61cc73854384f3e50f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/5c4f5de7563db2bee452a102614c53b8e… commit 5c4f5de7563db2bee452a102614c53b8ed9583ed Author: Haizhu Liu <haizhu(a)us.ibm.com> Date: Fri Mar 22 21:09:05 2013 -0400 Fix seg fault in _mpi_world_exiting_handler, fix hang at MPI_Finalize, removing calling PAMI_Context_advance in dispatch. (ibm) D189340 (ibm) 7c57f7ed7a8aa1f0156ba5112664a7c2cd35f227 Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> diff --git a/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c b/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c index 094ca0f..3e5378e 100644 --- a/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c +++ b/src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c @@ -137,29 +137,23 @@ void MPIDI_wait_for_AM(long long tranid, int expected_AM, int whichAM) double starttime, currtime, elapsetime; int rc, curr_AMcntr; - MPIU_THREAD_CS_EXIT(ALLFUNC,); rc = PAMI_Context_advance(MPIDI_Context[0], (size_t)100); - MPIU_THREAD_CS_ENTER(ALLFUNC,); if(whichAM == LAST_AM) { CURTIME(starttime) do { CURTIME(currtime) elapsetime = currtime - starttime; - MPIU_THREAD_CS_EXIT(ALLFUNC,); rc = PAMI_Context_advance(MPIDI_Context[0], (size_t)100); - MPIU_THREAD_CS_ENTER(ALLFUNC,); curr_AMcntr = MPIDI_get_AM_cntr_for_tranid(tranid, whichAM); - TRACE_ERR("_try_to_disconnect: Looping in timer for TranID %lld, whichAM %d expected_AM = %d, Current AM = %d\n",tranid,whichAM,expected_AM,curr_AMcntr); + /*TRACE_ERR("_try_to_disconnect: Looping in timer for TranID %lld, whichAM %d expected_AM = %d, Current AM = %d\n",tranid,whichAM,expected_AM,curr_AMcntr); */ }while(curr_AMcntr != expected_AM && elapsetime < DISCONNECT_LAPI_XFER_TIMEOUT); } else { do { - MPIU_THREAD_CS_EXIT(ALLFUNC,); rc = PAMI_Context_advance(MPIDI_Context[0], (size_t)100); - MPIU_THREAD_CS_ENTER(ALLFUNC,); curr_AMcntr = MPIDI_get_AM_cntr_for_tranid(tranid, whichAM); - TRACE_ERR("_try_to_disconnect: Looping in timer for TranID %lld, whichAM %d expected_AM = %d, Current AM = %d\n",tranid,whichAM,expected_AM,curr_AMcntr); + /*TRACE_ERR("_try_to_disconnect: Looping in timer for TranID %lld, whichAM %d expected_AM = %d, Current AM = %d\n",tranid,whichAM,expected_AM,curr_AMcntr);*/ }while(curr_AMcntr != expected_AM); } } diff --git a/src/mpid/pamid/src/dyntask/mpidi_pg.c b/src/mpid/pamid/src/dyntask/mpidi_pg.c index d588882..f59890d 100644 --- a/src/mpid/pamid/src/dyntask/mpidi_pg.c +++ b/src/mpid/pamid/src/dyntask/mpidi_pg.c @@ -170,9 +170,9 @@ int MPIDI_PG_Finalize(void) MPIU_Free(root_wid_barray); /* root_wid_barray is now NULL for non-root */ - +#if 0 pthread_create(&finalize_req_thread, NULL, mpidi_finalize_req, NULL); - MPIU_THREAD_CS_EXIT(ALLFUNC,); + /*MPIU_THREAD_CS_EXIT(ALLFUNC,); */ while (mpidi_sync_done !=1) { mpi_errno=PAMI_Context_advance(MPIDI_Context[0], 1000); if (mpi_errno == PAMI_EAGAIN) { @@ -183,6 +183,9 @@ int MPIDI_PG_Finalize(void) if (mpi_errno = pthread_join(finalize_req_thread, NULL) ) { TRACE_ERR("error returned from pthread_join() mpi_errno=%d\n",mpi_errno); } +#endif + MPIU_THREAD_CS_EXIT(ALLFUNC,); + PMI2_Finalize(); MPIU_THREAD_CS_ENTER(ALLFUNC,); if(_conn_info_list) { diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 022a2fd..4f85bd6 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -33,6 +33,8 @@ #define MAX_JOBID_LEN 1024 int world_rank; int world_size; +extern int (*mp_world_exiting_handler)(int); +extern int _mpi_world_exiting_handler(int); #endif int mpidi_dynamic_tasking = 0; @@ -1009,10 +1011,8 @@ MPIDI_VCRT_init(int rank, int size, char *world_tasks, MPIDI_PG_t *pg) world_tasks_save = MPIU_Strdup(world_tasks); if(world_tasks != NULL) { comm->vcr[0]->taskid = atoi(strtok(world_tasks, ":")); - TRACE_ERR("comm->vcr[0]->taskid =%d\n", comm->vcr[0]->taskid); while( (cp=strtok(NULL, ":")) != NULL) { comm->vcr[++i]->taskid= atoi(cp); - TRACE_ERR("comm->vcr[i]->taskid =%d\n", comm->vcr[i]->taskid); } } MPIU_Free(world_tasks_save); @@ -1228,6 +1228,7 @@ int MPID_Init(int * argc, /* FIXME: Check that this intercommunicator gets freed in MPI_Finalize if not already freed. */ } + mp_world_exiting_handler = &(_mpi_world_exiting_handler); #endif /* ------------------------------- */ /* Initialize timer data */ diff --git a/src/pmi/pmi2/poe/poe2pmi.c b/src/pmi/pmi2/poe/poe2pmi.c index e37563e..4fa1fa0 100644 --- a/src/pmi/pmi2/poe/poe2pmi.c +++ b/src/pmi/pmi2/poe/poe2pmi.c @@ -69,8 +69,7 @@ static MPID_Thread_cond_t cond; #endif extern int mpidi_finalized; -extern int (*mp_world_exiting_handler)(int); -extern int _mpi_world_exiting_handler(int); +int _mpi_world_exiting_handler(int); void *poeptr = NULL; @@ -98,7 +97,7 @@ int PMI2_Init(int *spawned, int *size, int *rank, int *appnum) } ret = (*pmi2_init)(spawned, size, rank, appnum); - mp_world_exiting_handler = &(_mpi_world_exiting_handler); + /*mp_world_exiting_handler = &(_mpi_world_exiting_handler); */ return ret; } @@ -271,7 +270,7 @@ int PMI2_Info_GetJobAttr(const char name[], char value[], int valuelen, int *fla * This is the mpi level of callback that get invoked when a task get notified * of a world's exiting */ -int _mpi_world_exiting_handler_wrapper(pami_context_t context, void *cookie) +int _mpi_world_exiting_handler(int world_id) { /* check the reference count associated with that remote world if the reference count is zero, the task will call LAPI_Purge_totask on @@ -286,10 +285,10 @@ int _mpi_world_exiting_handler_wrapper(pami_context_t context, void *cookie) char world_id_str[32]; int mpi_errno = MPI_SUCCESS; pami_endpoint_t dest; - struct worldExitReq *req = (struct worldExitReq *)cookie; - int world_id = req->world_id; +/* struct worldExitReq *req = (struct worldExitReq *)cookie; */ MPID_Comm *comm = MPIR_Process.comm_world; + MPIU_THREAD_CS_ENTER(ALLFUNC,); ref_count = MPIDI_get_refcnt_of_world(world_id); TRACE_ERR("_mpi_world_exiting_handler: invoked for world %d exiting ref_count=%d my comm_word_size=%d\n", world_id, ref_count, world_size); if(ref_count == 0) { @@ -300,7 +299,7 @@ int _mpi_world_exiting_handler_wrapper(pami_context_t context, void *cookie) MPIDI_OpState_reset(taskid_list[i]); MPIDI_IpState_reset(taskid_list[i]); TRACE_ERR("PAMI_Purge on taskid_list[%d]=%d\n", i,taskid_list[i]); - PAMI_Purge(context, &dest, 1); + PAMI_Purge(MPIDI_Context[0], &dest, 1); } MPIDI_delete_conn_record(world_id); } @@ -313,6 +312,7 @@ int _mpi_world_exiting_handler_wrapper(pami_context_t context, void *cookie) TRACE_ERR("_mpi_world_exiting_handler: Out of _mpi_reduce_for_dyntask for exiting world %d reduce_state=%d\n",world_id, reduce_state); + MPIU_THREAD_CS_EXIT(ALLFUNC,); if(comm->rank == 0) { MPIU_Snprintf(world_id_str, sizeof(world_id_str), "%d", world_id); PMI2_Abort(0, world_id_str); @@ -329,28 +329,11 @@ int _mpi_world_exiting_handler_wrapper(pami_context_t context, void *cookie) rc = -2; } - if(cookie) MPIU_Free(cookie); +/* if(cookie) MPIU_Free(cookie);*/ return PAMI_SUCCESS; } -int _mpi_world_exiting_handler(int world_id) -{ - struct worldExitReq *req; - req = MPIU_Malloc(sizeof(struct worldExitReq)); - req->world_id = world_id; - - if(MPIDI_Context[0]) { - if(!mpidi_finalized) - PAMI_Context_post(MPIDI_Context[0], &(req->work), _mpi_world_exiting_handler_wrapper, req); - else - _mpi_world_exiting_handler_wrapper(MPIDI_Context[0], req); - } - - return MPI_SUCCESS; -} - - int getchildren(int iam, double alpha,int gsize, int *children, int *blocks, int *numchildren, int *parent) { @@ -418,14 +401,17 @@ int _mpi_reduce_for_dyntask(int *sendbuf, int *recvbuf) { remaining_child_count = i; child_rank = (children[i])% TASKS; - mpi_errno = MPIC_Recv(recvbuf, sizeof(int),MPI_BYTE, pg_world->vct[child_rank].taskid, tag, comm_ptr->handle, MPI_STATUS_IGNORE); + TRACE_ERR("_mpi_reduce_for_dyntask - recv from child_rank%d child_taskid=%d\n", child_rank, pg_world->vct[child_rank].taskid); + mpi_errno = MPIC_Recv(recvbuf, sizeof(int),MPI_BYTE, child_rank, tag, comm_ptr->handle, MPI_STATUS_IGNORE); + TRACE_ERR("_mpi_reduce_for_dyntask - recv DONE from child_rank%d child_taskid=%d\n", child_rank, pg_world->vct[child_rank].taskid); if(world_rank != parent) { if(remaining_child_count == 0) { parent_rank = (parent) % TASKS; result += *recvbuf; - MPIC_Send(&result, sizeof(int), MPI_BYTE, pg_world->vct[parent_rank].taskid, tag, comm_ptr->handle); + TRACE_ERR("_mpi_reduce_for_dyntask - send to parent_rank=%d parent taskid=%d \n", parent_rank, pg_world->vct[parent_rank].taskid); + MPIC_Send(&result, sizeof(int), MPI_BYTE, parent_rank, tag, comm_ptr->handle); } else { @@ -440,7 +426,8 @@ int _mpi_reduce_for_dyntask(int *sendbuf, int *recvbuf) if(world_rank != parent && numchildren == 0) { parent_rank = (parent) % TASKS; - MPIC_Send(sendbuf, sizeof(int), MPI_BYTE, pg_world->vct[parent_rank].taskid, tag, comm_ptr->handle); + TRACE_ERR("_mpi_reduce_for_dyntask - send to parent_rank=%d parent_task_id=%d\n", parent_rank, pg_world->vct[parent_rank].taskid); + MPIC_Send(sendbuf, sizeof(int), MPI_BYTE, parent_rank, tag, comm_ptr->handle); } if(world_rank == 0) { ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/src/dyntask/mpid_comm_disconnect.c | 10 +---- src/mpid/pamid/src/dyntask/mpidi_pg.c | 7 ++- src/mpid/pamid/src/mpid_init.c | 5 +- src/pmi/pmi2/poe/poe2pmi.c | 43 +++++++------------- 4 files changed, 25 insertions(+), 40 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-164-g07de139
by noreply＠mpich.org 07 May '13

07 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 07de139fa7b3cba2894ddf61cc73854384f3e50f (commit) via bd899327f783a3a085f64e5830d8435dca22714d (commit) via 4450a0a2f0519d4f384da21b110ab7135d31791f (commit) from 058286b1b5308c965c4790878a49dbb3ffa1fcdf (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/07de139fa7b3cba2894ddf61cc7385438… commit 07de139fa7b3cba2894ddf61cc73854384f3e50f Author: James Dinan <dinan(a)mcs.anl.gov> Date: Tue May 7 14:22:48 2013 -0500 Remove unnecessary myrank field in MPID_Win Myrank was caching win_ptr->comm_ptr->rank, so we now use that directly rather than caching it in the MPID_Win object. Reviewer: balaji diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c index 6aa1729..09b101b 100644 --- a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c +++ b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c @@ -79,7 +79,7 @@ int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr) /* Free shared process mutex memory region */ if ((*win_ptr)->shm_mutex) { - if ((*win_ptr)->myrank == 0) { + if ((*win_ptr)->comm_ptr->rank == 0) { MPIDI_CH3I_SHM_MUTEX_DESTROY(*win_ptr); } diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 3a23e5b..62c78a9 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -257,8 +257,6 @@ struct MPIDI_Win_target_state { }; #define MPIDI_DEV_WIN_DECL \ - int myrank; /* Rank of this process in comm (used to \ - detect operations on self) */ \ volatile int my_counter; /* completion counter for operations \ targeting this window */ \ void **base_addrs; /* array of base addresses of the windows of \ diff --git a/src/mpid/ch3/src/ch3u_rma_acc_ops.c b/src/mpid/ch3/src/ch3u_rma_acc_ops.c index 29a46b1..f50e6b1 100644 --- a/src/mpid/ch3/src/ch3u_rma_acc_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_acc_ops.c @@ -52,7 +52,7 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count, goto fn_exit; } - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; origin_predefined = TRUE; /* quiet uninitialized warnings (b/c goto) */ if (op != MPI_NO_OP) { @@ -286,7 +286,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr, MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; /* The datatype must be predefined, and one of: C integer, Fortran integer, * Logical, Multi-language types, or Byte. This is checked above the ADI, @@ -393,7 +393,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr, MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; /* The datatype and op must be predefined. This is checked above the ADI, * so there's no need to check it again here. */ diff --git a/src/mpid/ch3/src/ch3u_rma_ops.c b/src/mpid/ch3/src/ch3u_rma_ops.c index f2fc7ca..95d47b9 100644 --- a/src/mpid/ch3/src/ch3u_rma_ops.c +++ b/src/mpid/ch3/src/ch3u_rma_ops.c @@ -207,7 +207,7 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype goto fn_exit; } - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; /* If the put is a local operation, do it here */ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) @@ -318,7 +318,7 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype goto fn_exit; } - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; /* If the get is a local operation, do it here */ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) @@ -428,7 +428,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype goto fn_exit; } - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, origin_predefined); MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined); diff --git a/src/mpid/ch3/src/ch3u_rma_reqops.c b/src/mpid/ch3/src/ch3u_rma_reqops.c index acfe1cb..71335b3 100644 --- a/src/mpid/ch3/src/ch3u_rma_reqops.c +++ b/src/mpid/ch3/src/ch3u_rma_reqops.c @@ -181,7 +181,7 @@ int MPIDI_Rput(const void *origin_addr, int origin_count, /* If the operation is already complete, return a completed request. * Otherwise, generate a grequest. */ /* FIXME: We still may need to flush or sync for shared memory windows */ - if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->myrank || + if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || data_sz == 0) { mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query, @@ -262,7 +262,7 @@ int MPIDI_Rget(void *origin_addr, int origin_count, /* If the operation is already complete, return a completed request. * Otherwise, generate a grequest. */ /* FIXME: We still may need to flush or sync for shared memory windows */ - if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->myrank || + if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || data_sz == 0) { mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query, @@ -342,7 +342,7 @@ int MPIDI_Raccumulate(const void *origin_addr, int origin_count, /* If the operation is already complete, return a completed request. * Otherwise, generate a grequest. */ /* FIXME: We still may need to flush or sync for shared memory windows */ - if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->myrank || + if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || data_sz == 0) { mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query, @@ -428,7 +428,7 @@ int MPIDI_Rget_accumulate(const void *origin_addr, int origin_count, /* If the operation is already complete, return a completed request. * Otherwise, generate a grequest. */ /* FIXME: We still may need to flush or sync for shared memory windows */ - if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->myrank || + if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || (data_sz == 0 && trg_data_sz == 0)) { diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index 00b0a76..1e7e4a6 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -1370,7 +1370,7 @@ int MPIDI_Win_post(MPID_Group *post_grp_ptr, int assert, MPID_Win *win_ptr) win_grp_ptr, ranks_in_win_grp); if (mpi_errno) MPIU_ERR_POP(mpi_errno); - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; MPIU_CHKLMEM_MALLOC(req, MPI_Request *, post_grp_size * sizeof(MPI_Request), mpi_errno, "req"); MPIU_CHKLMEM_MALLOC(status, MPI_Status *, post_grp_size*sizeof(MPI_Status), mpi_errno, "status"); @@ -1562,7 +1562,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr) win_grp_ptr, ranks_in_win_grp); if (mpi_errno) MPIU_ERR_POP(mpi_errno); - rank = win_ptr->myrank; + rank = win_ptr->comm_ptr->rank; /* If MPI_MODE_NOCHECK was not specified, we need to check if Win_post was called on the target processes. Wait for a 0-byte sync @@ -1920,7 +1920,7 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr) target_state->remote_lock_mode = lock_type; target_state->remote_lock_assert = assert; - if (dest == win_ptr->myrank) { + if (dest == win_ptr->comm_ptr->rank) { /* The target is this process itself. We must block until the lock * is acquired. Once it is acquired, local puts, gets, accumulates * will be done directly without queueing. */ @@ -1989,7 +1989,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr) OPA_read_write_barrier(); } - if (dest == win_ptr->myrank) { + if (dest == win_ptr->comm_ptr->rank) { /* local lock. release the lock on the window, grant the next one * in the queue, and return. */ MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->targets[dest].rma_ops_list)); @@ -2410,7 +2410,7 @@ int MPIDI_Win_lock_all(int assert, MPID_Win *win_ptr) for (i = 0; i < MPIR_Comm_size(win_ptr->comm_ptr); i++) { /* Local process is already locked */ - if (i == win_ptr->myrank) continue; + if (i == win_ptr->comm_ptr->rank) continue; mpi_errno = MPIDI_CH3I_Send_lock_msg(i, MPI_LOCK_SHARED, win_ptr); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -2418,7 +2418,7 @@ int MPIDI_Win_lock_all(int assert, MPID_Win *win_ptr) for (i = 0; i < MPIR_Comm_size(win_ptr->comm_ptr); i++) { /* Local process is already locked */ - if (i == win_ptr->myrank) continue; + if (i == win_ptr->comm_ptr->rank) continue; mpi_errno = MPIDI_CH3I_Wait_for_lock_granted(win_ptr, i); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -2772,8 +2772,8 @@ static int MPIDI_CH3I_Acquire_local_lock(MPID_Win *win_ptr, int lock_type) { MPIU_INSTR_DURATION_END(winlock_getlocallock); } - win_ptr->targets[win_ptr->myrank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; - win_ptr->targets[win_ptr->myrank].remote_lock_mode = lock_type; + win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_state = MPIDI_CH3_WIN_LOCK_GRANTED; + win_ptr->targets[win_ptr->comm_ptr->rank].remote_lock_mode = lock_type; fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_ACQUIRE_LOCAL_LOCK); diff --git a/src/mpid/ch3/src/mpid_rma.c b/src/mpid/ch3/src/mpid_rma.c index bb10dca..8a581e7 100644 --- a/src/mpid/ch3/src/mpid_rma.c +++ b/src/mpid/ch3/src/mpid_rma.c @@ -266,7 +266,6 @@ static int win_init(MPI_Aint size, int disp_unit, int create_flavor, int model, (*win_ptr)->start_group_ptr = NULL; (*win_ptr)->start_assert = 0; (*win_ptr)->comm_ptr = win_comm_ptr; - (*win_ptr)->myrank = comm_ptr->rank; (*win_ptr)->my_counter = 0; /* (*win_ptr)->base_addrs[] is set by caller; */ http://git.mpich.org/mpich.git/commitdiff/bd899327f783a3a085f64e5830d8435dc… commit bd899327f783a3a085f64e5830d8435dca22714d Author: James Dinan <dinan(a)mcs.anl.gov> Date: Tue May 7 12:54:13 2013 -0500 Move CH3-specific MPID_Win members into CH3 Several fields of MPID_Win were not used above the ADI. These have been moved to CH3 to reduce clutter. Downstream devices should redefine these fields in MPID_DEV_WIN_DECL if they are needed. Reviewer: balaji diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h index d3e6d03..deedd53 100644 --- a/src/include/mpiimpl.h +++ b/src/include/mpiimpl.h @@ -1693,10 +1693,7 @@ typedef struct MPID_Win { MPI_Aint size; int disp_unit; /* Displacement unit of *local* window */ MPID_Attribute *attributes; - MPID_Group *start_group_ptr; /* group passed in MPI_Win_start */ MPID_Comm *comm_ptr; /* Pointer to comm of window (dup) */ - int myrank; /* Rank of this process in comm (used to - detect operations on self) */ #ifdef USE_THREADED_WINDOW_CODE /* These were causing compilation errors. We need to figure out how to integrate threads into MPICH before including these fields. */ diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 49245d0..3a23e5b 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -257,6 +257,8 @@ struct MPIDI_Win_target_state { }; #define MPIDI_DEV_WIN_DECL \ + int myrank; /* Rank of this process in comm (used to \ + detect operations on self) */ \ volatile int my_counter; /* completion counter for operations \ targeting this window */ \ void **base_addrs; /* array of base addresses of the windows of \ @@ -293,6 +295,7 @@ struct MPIDI_Win_target_state { this state must be updated collectively (in \ fence) to ensure that the fence state across \ all processes remains consistent. */ \ + MPID_Group *start_group_ptr; /* group passed in MPI_Win_start */ \ int start_assert; /* assert passed to MPI_Win_start */ \ #ifdef MPIDI_CH3_WIN_DECL http://git.mpich.org/mpich.git/commitdiff/4450a0a2f0519d4f384da21b110ab7135… commit 4450a0a2f0519d4f384da21b110ab7135d31791f Author: James Dinan <dinan(a)mcs.anl.gov> Date: Tue May 7 13:29:13 2013 -0500 Cleanup RMA comment in MPIDI_Reqest diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 032670a..49245d0 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -370,7 +370,7 @@ typedef struct MPIDI_Request { * unexpected, exclusive access otherwise */ int recv_pending_count; - /* The next 8 are for RMA */ + /* The next several fields are used to hold state for ongoing RMA operations */ MPI_Op op; /* For accumulate, since data is first read into a tmp_buf */ void *real_user_buf; ----------------------------------------------------------------------- Summary of changes: src/include/mpiimpl.h | 3 --- src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c | 2 +- src/mpid/ch3/include/mpidpre.h | 3 ++- src/mpid/ch3/src/ch3u_rma_acc_ops.c | 6 +++--- src/mpid/ch3/src/ch3u_rma_ops.c | 6 +++--- src/mpid/ch3/src/ch3u_rma_reqops.c | 8 ++++---- src/mpid/ch3/src/ch3u_rma_sync.c | 16 ++++++++-------- src/mpid/ch3/src/mpid_rma.c | 1 - 8 files changed, 21 insertions(+), 24 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-161-g058286b
by noreply＠mpich.org 07 May '13

07 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 058286b1b5308c965c4790878a49dbb3ffa1fcdf (commit) from bbb5a3c4b26f9642fc1dec6c96c9c7ca1f695f17 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/058286b1b5308c965c4790878a49dbb3f… commit 058286b1b5308c965c4790878a49dbb3ffa1fcdf Author: William Gropp <wgropp(a)illinois.edu> Date: Tue May 7 14:25:13 2013 -0500 Add casts to suppress false warnings about alignment issues. diff --git a/src/mpl/src/mpltrmem.c b/src/mpl/src/mpltrmem.c index dc455ff..cac9f12 100644 --- a/src/mpl/src/mpltrmem.c +++ b/src/mpl/src/mpltrmem.c @@ -233,7 +233,8 @@ void *MPL_trmalloc(size_t a, int lineno, const char fname[]) goto fn_exit; memset(new, TRDefaultByte, nsize + sizeof(TrSPACE) + sizeof(unsigned long)); - head = (TRSPACE *) new; + /* Cast to (void*) to avoid false warnings about alignment issues */ + head = (TRSPACE *) (void *)new; new += sizeof(TrSPACE); if (TRhead[0] != TRHEAD_PRESENTINAL || TRhead[2] != TRHEAD_POSTSENTINAL) { @@ -256,7 +257,8 @@ void *MPL_trmalloc(size_t a, int lineno, const char fname[]) MPL_strncpy(head->fname, fname, TR_FNAME_LEN); head->fname[TR_FNAME_LEN - 1] = 0; head->cookie = COOKIE_VALUE; - nend = (unsigned long *) (new + nsize); + /* Cast to (void*) to avoid false warning about alignment */ + nend = (unsigned long *) (void *) (new + nsize); nend[0] = COOKIE_VALUE; allocated += nsize; @@ -309,7 +311,9 @@ void MPL_trfree(void *a_ptr, int line, const char file[]) return; } - head = (TRSPACE *) ( ((char *)a_ptr) - sizeof(TrSPACE) ); + /* Alignment guaranteed by the way a_ptr was allocated. Use + (void *) cast to suppress false warning about alignment issues */ + head = (TRSPACE *) (void *) ( ((char *)a_ptr) - sizeof(TrSPACE) ); /* We need to mark the memory as defined before performing our own error * checks or valgrind will flag the trfree function as erroneous. The real @@ -328,7 +332,8 @@ void MPL_trfree(void *a_ptr, int line, const char file[]) "called in %s at line %d\n", world_rank, hexstring, file, line); return; } - nend = (unsigned long *) ((char *)a_ptr + head->size); + /* Cast to (void*) to avoid false warning about alignment */ + nend = (unsigned long *) (void *) ((char *)a_ptr + head->size); /* Check that nend is properly aligned */ if ((sizeof(long) == 4 && ((long) nend & 0x3) != 0) || (sizeof(long) == 8 && ((long) nend & 0x7) != 0)) { @@ -506,7 +511,8 @@ int MPL_trvalid2(const char str[], int line, const char file[] ) the full header is padded to ensure correct byte alignment with the data */ a = (char *)( (TrSPACE *)head + 1 ); - nend = (unsigned long *) (a + head->size); + /* Cast to (void*) to avoid false warning about alignment */ + nend = (unsigned long *) (void *)(a + head->size); /* mark defined before accessing nend contents */ MPL_VG_MAKE_MEM_DEFINED(nend, sizeof(*nend)); @@ -785,9 +791,9 @@ void *MPL_trrealloc(void *p, size_t size, int lineno, const char fname[]) TRSPACE *head = 0; char hexstring[MAX_ADDRESS_CHARS]; -/* We should really use the size of the old block... */ + /* We should really use the size of the old block... */ if (p) { - head = (TRSPACE *) ((char *)p - sizeof(TrSPACE)); + head = (TRSPACE *) (void*) ((char *)p - sizeof(TrSPACE)); MPL_VG_MAKE_MEM_DEFINED(head, sizeof(*head)); /* mark defined before accessing contents */ if (head->cookie != COOKIE_VALUE) { /* Damaged header */ diff --git a/src/util/mem/handlemem.c b/src/util/mem/handlemem.c index 61ed867..7894746 100644 --- a/src/util/mem/handlemem.c +++ b/src/util/mem/handlemem.c @@ -157,7 +157,9 @@ void *MPIU_Handle_direct_init(void *direct, for (i=0; i<direct_size; i++) { /* printf( "Adding %p in %d\n", ptr, handle_type ); */ - hptr = (MPIU_Handle_common *)ptr; + /* First cast to (void*) to avoid false warnings about alignment + (consider that a requirement of the input parameters) */ + hptr = (MPIU_Handle_common *)(void *)ptr; ptr = ptr + obj_size; hptr->next = ptr; hptr->handle = ((unsigned)HANDLE_KIND_DIRECT << HANDLE_KIND_SHIFT) | @@ -209,7 +211,8 @@ static void *MPIU_Handle_indirect_init( void *(**indirect)[], } ptr = (char *)block_ptr; for (i=0; i<indirect_num_indices; i++) { - hptr = (MPIU_Handle_common *)ptr; + /* Cast to (void*) to avoid false warning about alignment */ + hptr = (MPIU_Handle_common *)(void*)ptr; ptr = ptr + obj_size; hptr->next = ptr; hptr->handle = ((unsigned)HANDLE_KIND_INDIRECT << HANDLE_KIND_SHIFT) | ----------------------------------------------------------------------- Summary of changes: src/mpl/src/mpltrmem.c | 20 +++++++++++++------- src/util/mem/handlemem.c | 7 +++++-- 2 files changed, 18 insertions(+), 9 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.0.4-160-gbbb5a3c
by noreply＠mpich.org 07 May '13

07 May '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via bbb5a3c4b26f9642fc1dec6c96c9c7ca1f695f17 (commit) via e50a1dd3dd9460f0c7c23b63fb74162068b98593 (commit) via dd85044ba8d3cf760e4138a1f6012c45e54547af (commit) via 313640c691ee440e85ab2a04375c804031fe90db (commit) via f4f0de65736a8abf6c96b727e87b082992616cf5 (commit) via 5673b82ca24c7481e312dc7573c84811c8244bb9 (commit) via a28689d4ff7104ba525ddc5ea512e2d45116c1ce (commit) from 1be61fbcad263cffe6dedfb1597be46ccad2b714 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/bbb5a3c4b26f9642fc1dec6c96c9c7ca1… commit bbb5a3c4b26f9642fc1dec6c96c9c7ca1f695f17 Author: sssharka <sssharka(a)us.ibm.com> Date: Fri Mar 29 10:32:24 2013 -0400 Fixes for parsing command line parameters in CollSel (ibm) D189463 (ibm) e7be2d5b3ae75a32383c32eb203d864d1bbbc386 Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c index f1b508c..5f9f2cf 100644 --- a/src/mpid/pamid/src/mpidi_util.c +++ b/src/mpid/pamid/src/mpidi_util.c @@ -1404,6 +1404,7 @@ static int MPIDI_collsel_process_msg_sizes(char *msg_sizes_arg, advisor_params_t char *msg_sizes = (char *) MPIU_Malloc(arg_len + 1); char *msg_sz; size_t tmp; + size_t array_size = 50; /* if already set via config file, free it */ if(params->message_sizes) { @@ -1411,7 +1412,7 @@ static int MPIDI_collsel_process_msg_sizes(char *msg_sizes_arg, advisor_params_t params->num_message_sizes = 0; } /* Allocating some extra space should be fine */ - params->message_sizes = (size_t *)MPIU_Malloc(sizeof(size_t) * 50); + params->message_sizes = (size_t *)MPIU_Malloc(sizeof(size_t) * array_size); strcpy(msg_sizes, msg_sizes_arg); msg_sz = strtok(msg_sizes,","); @@ -1430,6 +1431,12 @@ static int MPIDI_collsel_process_msg_sizes(char *msg_sizes_arg, advisor_params_t break; } + if(params->num_message_sizes >= array_size) + { + array_size *= 2; + params->message_sizes = (size_t *) MPIU_Realloc(params->message_sizes, + sizeof(size_t) * array_size); + } params->message_sizes[params->num_message_sizes++] = tmp; msg_sz = strtok(NULL,","); } @@ -1444,6 +1451,7 @@ static int MPIDI_collsel_process_geo_sizes(char *geo_sizes_arg, advisor_params_t char *geo_sizes = (char *) MPIU_Malloc(arg_len + 1); char *geo_sz; size_t tmp; + size_t array_size = 50; /* if already set via config file, free it */ if(params->geometry_sizes) { @@ -1451,7 +1459,7 @@ static int MPIDI_collsel_process_geo_sizes(char *geo_sizes_arg, advisor_params_t params->num_geometry_sizes = 0; } /* Allocating some extra space should be fine */ - params->geometry_sizes = (size_t *)MPIU_Malloc(sizeof(size_t) * 50); + params->geometry_sizes = (size_t *)MPIU_Malloc(sizeof(size_t) * array_size); strcpy(geo_sizes, geo_sizes_arg); geo_sz = strtok(geo_sizes,","); @@ -1470,6 +1478,12 @@ static int MPIDI_collsel_process_geo_sizes(char *geo_sizes_arg, advisor_params_t break; } + if(params->num_geometry_sizes >= array_size) + { + array_size *= 2; + params->geometry_sizes = (size_t *) MPIU_Realloc(params->geometry_sizes, + sizeof(size_t) * array_size); + } params->geometry_sizes[params->num_geometry_sizes++] = tmp; geo_sz = strtok(NULL,","); } http://git.mpich.org/mpich.git/commitdiff/e50a1dd3dd9460f0c7c23b63fb7416206… commit e50a1dd3dd9460f0c7c23b63fb74162068b98593 Author: sssharka <sssharka(a)us.ibm.com> Date: Wed Mar 20 19:14:25 2013 -0400 shmcc_gather_t2 coredump w/ setMP_COLLECTIVE_SELECTION_FILE (ibm) D189345 (ibm) 7Z8 (ibm) 9e1fffe10532d58f09acdbbd81ee582b370bf804 Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/coll/allgather/mpido_allgather.c b/src/mpid/pamid/src/coll/allgather/mpido_allgather.c index 40f1de7..49e6aef 100644 --- a/src/mpid/pamid/src/coll/allgather/mpido_allgather.c +++ b/src/mpid/pamid/src/coll/allgather/mpido_allgather.c @@ -596,6 +596,7 @@ MPIDO_Allgather_simple(const void *sendbuf, MPID_Segment segment; volatile unsigned allgather_active = 1; const int rank = comm_ptr->rank; + const int size = comm_ptr->local_size; const pami_metadata_t *my_md; @@ -634,7 +635,7 @@ MPIDO_Allgather_simple(const void *sendbuf, if(!rcv_data_contig) { - rcv_noncontig_buff = MPIU_Malloc(recv_size); + rcv_noncontig_buff = MPIU_Malloc(recv_size * size); rbuf = rcv_noncontig_buff; if(rcv_noncontig_buff == NULL) { @@ -693,7 +694,7 @@ MPIDO_Allgather_simple(const void *sendbuf, MPID_PROGRESS_WAIT_WHILE(allgather_active); if(!rcv_data_contig) { - MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, + MPIR_Localcopy(rcv_noncontig_buff, recv_size * size, MPI_CHAR, recvbuf, recvcount, recvtype); MPIU_Free(rcv_noncontig_buff); } diff --git a/src/mpid/pamid/src/coll/gather/mpido_gather.c b/src/mpid/pamid/src/coll/gather/mpido_gather.c index 1226cae..1047a44 100644 --- a/src/mpid/pamid/src/coll/gather/mpido_gather.c +++ b/src/mpid/pamid/src/coll/gather/mpido_gather.c @@ -453,7 +453,7 @@ int MPIDO_Gather_simple(const void *sendbuf, rbuf = (char *)recvbuf + true_lb; if (!rcv_contig) { - rcv_noncontig_buff = MPIU_Malloc(recv_size); + rcv_noncontig_buff = MPIU_Malloc(recv_size * size); rbuf = rcv_noncontig_buff; if(rcv_noncontig_buff == NULL) { @@ -506,8 +506,8 @@ int MPIDO_Gather_simple(const void *sendbuf, if(!rcv_contig) { - MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, - recvbuf, recvcount, recvtype); + MPIR_Localcopy(rcv_noncontig_buff, recv_size*size, MPI_CHAR, + recvbuf, recvcount*size, recvtype); MPIU_Free(rcv_noncontig_buff); } if(!snd_contig) MPIU_Free(snd_noncontig_buff); diff --git a/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c b/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c index ed1034b..34b77bf 100644 --- a/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c +++ b/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c @@ -267,7 +267,7 @@ int MPIDO_Gatherv_simple(const void *sendbuf, if(MPIDI_Pamix_collsel_advise != NULL) { advisor_algorithm_t advisor_algorithms[1]; - int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, send_size, advisor_algorithms, 1); + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, 64, advisor_algorithms, 1); if(num_algorithms) { if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) @@ -301,7 +301,7 @@ int MPIDO_Gatherv_simple(const void *sendbuf, if(MPIDI_Pamix_collsel_advise != NULL) { advisor_algorithm_t advisor_algorithms[1]; - int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, rcvlen * recvcounts[0], advisor_algorithms, 1); + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, 64, advisor_algorithms, 1); if(num_algorithms) { if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) diff --git a/src/mpid/pamid/src/coll/scatter/mpido_scatter.c b/src/mpid/pamid/src/coll/scatter/mpido_scatter.c index a511f96..15d8366 100644 --- a/src/mpid/pamid/src/coll/scatter/mpido_scatter.c +++ b/src/mpid/pamid/src/coll/scatter/mpido_scatter.c @@ -335,7 +335,7 @@ int MPIDO_Scatter_simple(const void *sendbuf, int use_pami = 1; MPID_Segment segment; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); - + const int size = comm_ptr->local_size; if (rank == root && sendtype != MPI_DATATYPE_NULL && sendcount >= 0) { @@ -376,7 +376,6 @@ int MPIDO_Scatter_simple(const void *sendbuf, } } } - sbuf = (char *)sendbuf + snd_true_lb; rbuf = (char *)recvbuf + rcv_true_lb; if (rank == root) @@ -386,15 +385,15 @@ int MPIDO_Scatter_simple(const void *sendbuf, sbuf = (char *)sendbuf + snd_true_lb; if (!snd_contig) { - snd_noncontig_buff = MPIU_Malloc(send_size); + snd_noncontig_buff = MPIU_Malloc(send_size * size); sbuf = snd_noncontig_buff; if(snd_noncontig_buff == NULL) { MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, "Fatal: Cannot allocate pack buffer"); } - DLOOP_Offset last = send_size; - MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0); + DLOOP_Offset last = send_size * size; + MPID_Segment_init(sendbuf, sendcount * size, sendtype, &segment, 0); MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff); } } @@ -480,7 +479,7 @@ int MPIDO_Scatter_simple(const void *sendbuf, TRACE_ERR("Waiting on active %d\n", scatter_active); MPID_PROGRESS_WAIT_WHILE(scatter_active); - if(!rcv_contig) + if(!rcv_contig && rcv_noncontig_buff) { MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, recvbuf, recvcount, recvtype); diff --git a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c index f868c00..00946a3 100644 --- a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c +++ b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c @@ -468,7 +468,7 @@ int MPIDO_Scatterv_simple(const void *sendbuf, if(MPIDI_Pamix_collsel_advise != NULL) { advisor_algorithm_t advisor_algorithms[1]; - int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, ssize * sendcounts[0], advisor_algorithms, 1); + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, 64, advisor_algorithms, 1); if(num_algorithms) { if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) @@ -488,7 +488,7 @@ int MPIDO_Scatterv_simple(const void *sendbuf, if(MPIDI_Pamix_collsel_advise != NULL) { advisor_algorithm_t advisor_algorithms[1]; - int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, recv_size, advisor_algorithms, 1); + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, 64, advisor_algorithms, 1); if(num_algorithms) { if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) http://git.mpich.org/mpich.git/commitdiff/dd85044ba8d3cf760e4138a1f6012c45e… commit dd85044ba8d3cf760e4138a1f6012c45e54547af Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Mon May 6 13:54:41 2013 -0500 pami_tune core w/ -collective_selection=tune (ibm) D189342 (ibm) 7Z8 (ibm) c2e13cd5979dbba4019cb95c01688a629b7b0f9e Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index 1de14a0..3201595 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -392,7 +392,7 @@ void MPIDI_Coll_comm_destroy(MPID_Comm *comm) } - if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE) + if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE && MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_TUNE) { /* Destroy the fast query object. */ pami_extension_collsel_query_destroy pamix_collsel_query_destroy = diff --git a/src/mpid/pamid/src/comm/mpid_selectcolls.c b/src/mpid/pamid/src/comm/mpid_selectcolls.c index 886dbd8..cd30053 100644 --- a/src/mpid/pamid/src/comm/mpid_selectcolls.c +++ b/src/mpid/pamid/src/comm/mpid_selectcolls.c @@ -479,7 +479,7 @@ void MPIDI_Comm_coll_envvars(MPID_Comm *comm) /* If automatic collective selection is enabled and user didn't specifically overwrite it, then use auto coll sel.. Otherwise, go through the manual coll sel code path. */ - if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE) + if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE && MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_TUNE) { /* Create a fast query object, cache it on the comm/geometry and use it in each collective */ pami_extension_collsel_query_create pamix_collsel_query_create = http://git.mpich.org/mpich.git/commitdiff/313640c691ee440e85ab2a04375c80403… commit 313640c691ee440e85ab2a04375c804031fe90db Author: sssharka <sssharka(a)us.ibm.com> Date: Wed Feb 27 22:36:39 2013 -0500 shmcc_gather_t2 hang w/o MP_COLLECTIVE_SELECTION_FILE set This commit includes a fix for all rooted collectives where type info is local. The fix makes sure that non-contig data is converted and packed on all tasks and then calling PAMI. Another change in this commit is related to performance. MPICH2 queries the collsel extension for best algorithm before attempting any conversions. If algorithm is mpich, then it is called directly instead of going through the collsel extension path in PAMI. (ibm) D187892 (ibm) 7Z8 (ibm) dad6ba8f7883508cd7f83433232b8dadb6ccddcb Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index dc87146..42f0da0 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -292,7 +292,7 @@ struct MPIDI_Request #endif }; - +typedef void* fast_query_t; /** \brief This defines the portion of MPID_Comm that is specific to the Device */ struct MPIDI_Comm { @@ -351,6 +351,7 @@ struct MPIDI_Comm long long world_intercomm_cntr; int *world_ids; /* ids of worlds that composed this communicator (inter communicator created for dynamic tasking */ #endif + fast_query_t collsel_fast_query; }; @@ -557,4 +558,18 @@ typedef pami_result_t (*pami_extension_collsel_table_generate) (advisor_t, external_geometry_ops_t *, int); +typedef pami_result_t (*pami_extension_collsel_query_create) (advisor_table_t advisor_table, + pami_geometry_t geometry, + fast_query_t *query); + +typedef pami_result_t (*pami_extension_collsel_query_destroy) (fast_query_t *query); + +typedef int (*pami_extension_collsel_advise) (fast_query_t fast_query, + pami_xfer_type_t xfer_type, + size_t message_size, + advisor_algorithm_t algorithms_optimized[], + size_t max_algorithms); + + + #endif diff --git a/src/mpid/pamid/include/mpidi_externs.h b/src/mpid/pamid/include/mpidi_externs.h index f9a9683..774e5c4 100644 --- a/src/mpid/pamid/include/mpidi_externs.h +++ b/src/mpid/pamid/include/mpidi_externs.h @@ -38,6 +38,6 @@ extern advisor_table_t MPIDI_Collsel_advisor_table; extern pami_extension_t MPIDI_Collsel_extension; extern advisor_params_t MPIDI_Collsel_advisor_params; extern char *MPIDI_Collsel_output_file; - +extern pami_extension_collsel_advise MPIDI_Pamix_collsel_advise; #endif diff --git a/src/mpid/pamid/src/coll/allgather/mpido_allgather.c b/src/mpid/pamid/src/coll/allgather/mpido_allgather.c index bb9ecf3..40f1de7 100644 --- a/src/mpid/pamid/src/coll/allgather/mpido_allgather.c +++ b/src/mpid/pamid/src/coll/allgather/mpido_allgather.c @@ -590,7 +590,7 @@ MPIDO_Allgather_simple(const void *sendbuf, void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; MPI_Aint send_true_lb = 0; MPI_Aint recv_true_lb = 0; - int snd_data_contig = 1, rcv_data_contig; + int snd_data_contig = 1, rcv_data_contig = 1; size_t send_size = 0; size_t recv_size = 0; MPID_Segment segment; @@ -614,6 +614,22 @@ MPIDO_Allgather_simple(const void *sendbuf, recv_true_lb); send_size = recv_size; + + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLGATHER, send_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Allgather(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, + comm_ptr, mpierrno); + } + } + } + rbuf = (char *)recvbuf+recv_true_lb; if(!rcv_data_contig) diff --git a/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c b/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c index fbf826f..95874b9 100644 --- a/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c +++ b/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c @@ -637,6 +637,29 @@ MPIDO_Allgatherv_simple(const void *sendbuf, int tmp; const pami_metadata_t *my_md; + + MPIDI_Datatype_get_info(1, + recvtype, + rcv_data_contig, + recv_size, + dt_null, + recv_true_lb); + + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLGATHERV_INT, recv_size * recvcounts[0], advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Allgatherv(sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + comm_ptr, mpierrno); + } + } + } + if((sendbuf != MPI_IN_PLACE) && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)) { return MPIR_Allgatherv(sendbuf, sendcount, sendtype, @@ -649,12 +672,7 @@ MPIDO_Allgatherv_simple(const void *sendbuf, recvbuf, recvcounts, displs, recvtype, comm_ptr, mpierrno); } - MPIDI_Datatype_get_info(1, - recvtype, - rcv_data_contig, - recv_size, - dt_null, - recv_true_lb); + if(sendbuf == MPI_IN_PLACE) { diff --git a/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c b/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c index ff7ca70..b948b8e 100644 --- a/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c +++ b/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c @@ -419,6 +419,24 @@ int MPIDO_Allreduce_simple(const void *sendbuf, MPI_Aint data_true_lb = 0; int data_size, data_contig; + MPIDI_Datatype_get_info(1, dt, + data_contig, data_size, data_ptr, data_true_lb); + + + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLREDUCE, data_size * count, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Allreduce(sendbuf, recvbuf, count, dt, op, comm_ptr, mpierrno); + } + } + } + + rc = MPIDI_Datatype_to_pami(dt, &pdt, op, &pop, &mu); /* convert to metadata query */ @@ -428,8 +446,6 @@ int MPIDO_Allreduce_simple(const void *sendbuf, MPIDI_Update_last_algorithm(comm_ptr, "ALLREDUCE_MPICH"); return MPIR_Allreduce(sendbuf, recvbuf, count, dt, op, comm_ptr, mpierrno); } - MPIDI_Datatype_get_info(1, dt, - data_contig, data_size, data_ptr, data_true_lb); if(!data_contig) { diff --git a/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c b/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c index ba4f2a0..3464c82 100644 --- a/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c +++ b/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c @@ -223,42 +223,76 @@ int MPIDO_Alltoall_simple(const void *sendbuf, { TRACE_ERR("Entering MPIDO_Alltoall_optimized\n"); volatile unsigned active = 1; + void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; + void *sbuf = NULL, *rbuf = NULL; + size_t send_size = 0; + size_t recv_size = 0; + MPID_Segment segment; MPID_Datatype *sdt, *rdt; pami_type_t stype = NULL, rtype; MPI_Aint sdt_true_lb=0, rdt_true_lb; MPIDI_Post_coll_t alltoall_post; - int sndlen, rcvlen, snd_contig, rcv_contig, pamidt=1; + int sndlen, rcvlen, snd_contig = 1, rcv_contig = 1, pamidt=1; int tmp; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); + MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvlen, rdt, rdt_true_lb); + rbuf = (char *)recvbuf + rdt_true_lb; + recv_size = rcvlen * recvcount; + + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLTOALL, recv_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Alltoall_intra(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, + comm_ptr, mpierrno); + } + } + } + if(sendbuf != MPI_IN_PLACE) { MPIDI_Datatype_get_info(1, sendtype, snd_contig, sndlen, sdt, sdt_true_lb); - if(!snd_contig) pamidt = 0; + sbuf = (char *)sendbuf + sdt_true_lb; + send_size = sndlen * sendcount; + if(!snd_contig) + { + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + DLOOP_Offset last = send_size; + MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0); + MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff); + + } + } + + if(!rcv_contig) + { + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } } - MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvlen, rdt, rdt_true_lb); - if(!rcv_contig) pamidt = 0; /* Alltoall is much simpler if bytes are required because we don't need to * malloc displ/count arrays and copy things */ - /* Is it a built in type? If not, send to MPICH */ - if(sendbuf != MPI_IN_PLACE && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)) - pamidt = 0; - if(MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS) - pamidt = 0; - - if(pamidt == 0) - { - return MPIR_Alltoall_intra(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm_ptr, mpierrno); - - } - pami_xfer_t alltoall; const pami_metadata_t *my_alltoall_md; my_alltoall_md = &mpid->coll_metadata[PAMI_XFER_ALLTOALL][0][0]; @@ -269,17 +303,17 @@ int MPIDO_Alltoall_simple(const void *sendbuf, alltoall.cb_done = cb_alltoall; alltoall.cookie = (void *)&active; alltoall.algorithm = mpid->coll_algorithm[PAMI_XFER_ALLTOALL][0][0]; - alltoall.cmd.xfer_alltoall.stype = stype;/* stype is ignored when sndbuf == PAMI_IN_PLACE */ - alltoall.cmd.xfer_alltoall.stypecount = sendcount; - alltoall.cmd.xfer_alltoall.sndbuf = (char *)sendbuf + sdt_true_lb; + alltoall.cmd.xfer_alltoall.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */ + alltoall.cmd.xfer_alltoall.stypecount = send_size; + alltoall.cmd.xfer_alltoall.sndbuf = sbuf; if(sendbuf == MPI_IN_PLACE) { alltoall.cmd.xfer_alltoall.sndbuf = PAMI_IN_PLACE; } - alltoall.cmd.xfer_alltoall.rcvbuf = (char *)recvbuf + rdt_true_lb; - alltoall.cmd.xfer_alltoall.rtypecount = recvcount; - alltoall.cmd.xfer_alltoall.rtype = rtype; + alltoall.cmd.xfer_alltoall.rcvbuf = rbuf; + alltoall.cmd.xfer_alltoall.rtypecount = recv_size; + alltoall.cmd.xfer_alltoall.rtype = PAMI_TYPE_BYTE; MPIDI_Context_post(MPIDI_Context[0], &alltoall_post.state, MPIDI_Pami_post_wrapper, (void *)&alltoall); @@ -287,6 +321,14 @@ int MPIDO_Alltoall_simple(const void *sendbuf, TRACE_ERR("Waiting on active\n"); MPID_PROGRESS_WAIT_WHILE(active); + if(!rcv_contig) + { + MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, + recvbuf, recvcount, recvtype); + MPIU_Free(rcv_noncontig_buff); + } + if(!snd_contig) MPIU_Free(snd_noncontig_buff); + TRACE_ERR("Leaving MPIDO_Alltoall_optimized\n"); return MPI_SUCCESS; } diff --git a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c index e48ec66..646b89f 100644 --- a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c +++ b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c @@ -236,6 +236,28 @@ int MPIDO_Alltoallv_simple(const void *sendbuf, const int rank = comm_ptr->rank; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); + /* We don't pack and unpack in alltoallv as we do in alltoall because alltoallv has + more overhead of book keeping for sendcounts/displs and recvcounts/displs. Since, + alltoallv is non-rooted and all tasks has type info, decision to punt to mpich + will be the same on all tasks */ + + + + MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvtypelen, rdt, rdt_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_ALLTOALLV_INT, rcvtypelen * recvcounts[0], advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Alltoallv(sendbuf, sendcounts, senddispls, sendtype, + recvbuf, recvcounts, recvdispls, recvtype, + comm_ptr, mpierrno); + } + } + } if(sendbuf != MPI_IN_PLACE && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)) pamidt = 0; @@ -248,7 +270,6 @@ int MPIDO_Alltoallv_simple(const void *sendbuf, MPIDI_Datatype_get_info(1, sendtype, snd_contig, sndtypelen, sdt, sdt_true_lb); if(!snd_contig) pamidt = 0; } - MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rcvtypelen, rdt, rdt_true_lb); if(!rcv_contig) pamidt = 0; diff --git a/src/mpid/pamid/src/coll/bcast/mpido_bcast.c b/src/mpid/pamid/src/coll/bcast/mpido_bcast.c index 45450fb..1c943bd 100644 --- a/src/mpid/pamid/src/coll/bcast/mpido_bcast.c +++ b/src/mpid/pamid/src/coll/bcast/mpido_bcast.c @@ -294,6 +294,18 @@ int MPIDO_Bcast_simple(void *buffer, int data_size_one; MPIDI_Datatype_get_info(1, datatype, data_contig, data_size_one, data_ptr, data_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_BROADCAST, data_size_one * count, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Bcast_intra(buffer, count, datatype, root, comm_ptr, mpierrno); + } + } + } const int data_size = data_size_one*(size_t)count; diff --git a/src/mpid/pamid/src/coll/gather/mpido_gather.c b/src/mpid/pamid/src/coll/gather/mpido_gather.c index b0e1ca2..1226cae 100644 --- a/src/mpid/pamid/src/coll/gather/mpido_gather.c +++ b/src/mpid/pamid/src/coll/gather/mpido_gather.c @@ -379,34 +379,96 @@ int MPIDO_Gather_simple(const void *sendbuf, MPI_Aint true_lb = 0; pami_xfer_t gather; MPIDI_Post_coll_t gather_post; - int success = 1, contig, send_bytes=-1, recv_bytes = 0; + int success = 1, snd_contig = 1, rcv_contig = 1; + void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; + void *sbuf = NULL, *rbuf = NULL; + int send_size = 0; + int recv_size = 0; + MPID_Segment segment; const int rank = comm_ptr->rank; const int size = comm_ptr->local_size; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); if(sendbuf != MPI_IN_PLACE) { - MPIDI_Datatype_get_info(sendcount, sendtype, contig, - send_bytes, data_ptr, true_lb); - if (!contig) - success = 0; + MPIDI_Datatype_get_info(sendcount, sendtype, snd_contig, + send_size, data_ptr, true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHER, send_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Gather(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, + root, comm_ptr, mpierrno); + } + } + } + + sbuf = (char *)sendbuf + true_lb; + if (!snd_contig) + { + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + DLOOP_Offset last = send_size; + MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0); + MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff); + } + } + else + { + if(MPIDI_Pamix_collsel_advise != NULL) + { + MPIDI_Datatype_get_info(recvcount, recvtype, rcv_contig, + recv_size, data_ptr, true_lb); + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHER, recv_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Gather(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, + root, comm_ptr, mpierrno); + } + } + } } - if (success && rank == root) + rbuf = (char *)recvbuf + true_lb; + if (rank == root) { if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0) { - MPIDI_Datatype_get_info(recvcount, recvtype, contig, - recv_bytes, data_ptr, true_lb); - if (!contig) success = 0; + MPIDI_Datatype_get_info(recvcount, recvtype, rcv_contig, + recv_size, data_ptr, true_lb); + rbuf = (char *)recvbuf + true_lb; + if (!rcv_contig) + { + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + } } else success = 0; } - MPIDI_Update_last_algorithm(comm_ptr, "GATHER_MPICH"); if(!success) { + MPIDI_Update_last_algorithm(comm_ptr, "GATHER_MPICH"); return MPIR_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, mpierrno); @@ -419,23 +481,22 @@ int MPIDO_Gather_simple(const void *sendbuf, gather.cb_done = cb_gather; gather.cookie = (void *)&active; gather.cmd.xfer_gather.root = MPID_VCR_GET_LPID(comm_ptr->vcr, root); - gather.cmd.xfer_gather.stypecount = send_bytes;/* stypecount is ignored when sndbuf == PAMI_IN_PLACE */ - gather.cmd.xfer_gather.sndbuf = (void *)sendbuf; + gather.cmd.xfer_gather.stypecount = send_size;/* stypecount is ignored when sndbuf == PAMI_IN_PLACE */ + gather.cmd.xfer_gather.sndbuf = (void *)sbuf; if(sendbuf == MPI_IN_PLACE) { gather.cmd.xfer_gather.sndbuf = PAMI_IN_PLACE; } gather.cmd.xfer_gather.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */ - gather.cmd.xfer_gather.rcvbuf = (void *)recvbuf; + gather.cmd.xfer_gather.rcvbuf = (void *)rbuf; gather.cmd.xfer_gather.rtype = PAMI_TYPE_BYTE; - gather.cmd.xfer_gather.rtypecount = recv_bytes; + gather.cmd.xfer_gather.rtypecount = recv_size; gather.algorithm = mpid->coll_algorithm[PAMI_XFER_GATHER][0][0]; my_gather_md = &mpid->coll_metadata[PAMI_XFER_GATHER][0][0]; MPIDI_Update_last_algorithm(comm_ptr, my_gather_md->name); - TRACE_ERR("%s gather\n", MPIDI_Process.context_post.active>0?"Posting":"Invoking"); MPIDI_Context_post(MPIDI_Context[0], &gather_post.state, MPIDI_Pami_post_wrapper, (void *)&gather); @@ -443,6 +504,14 @@ int MPIDO_Gather_simple(const void *sendbuf, TRACE_ERR("Waiting on active: %d\n", active); MPID_PROGRESS_WAIT_WHILE(active); + if(!rcv_contig) + { + MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, + recvbuf, recvcount, recvtype); + MPIU_Free(rcv_noncontig_buff); + } + if(!snd_contig) MPIU_Free(snd_noncontig_buff); + TRACE_ERR("Leaving MPIDO_Gather_optimized\n"); return MPI_SUCCESS; } @@ -473,6 +542,7 @@ MPIDO_CSWrapper_gather(pami_xfer_t *gather, gather->cmd.xfer_gather.rcvbuf, gather->cmd.xfer_gather.rtypecount, recvtype, gather->cmd.xfer_gather.root, comm_ptr, &mpierrno); + if(gather->cb_done && rc == 0) gather->cb_done(NULL, gather->cookie, PAMI_SUCCESS); return rc; diff --git a/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c b/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c index d04b567..ed1034b 100644 --- a/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c +++ b/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c @@ -239,74 +239,141 @@ int MPIDO_Gatherv_simple(const void *sendbuf, { TRACE_ERR("Entering MPIDO_Gatherv_optimized\n"); - int rc; - int contig, rsize=0, ssize=0; - int pamidt = 1; - MPID_Datatype *dt_ptr = NULL; - MPI_Aint send_true_lb, recv_true_lb; - char *sbuf, *rbuf; - pami_type_t stype = NULL, rtype; - int tmp; + int snd_contig = 1, rcv_contig = 1; + void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; + void *sbuf = NULL, *rbuf = NULL; + int *rcounts = NULL; + int *rdispls = NULL; + int send_size = 0; + int recv_size = 0; + int rcvlen = 0; + int rcvcount = 0; + pami_type_t rtype = PAMI_TYPE_NULL; + MPID_Segment segment; + MPID_Datatype *data_ptr = NULL; + int send_true_lb, recv_true_lb = 0; + int i, tmp; volatile unsigned gatherv_active = 1; const int rank = comm_ptr->rank; + const int size = comm_ptr->local_size; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); - /* Check for native PAMI types and MPI_IN_PLACE on sendbuf */ - /* MPI_IN_PLACE is a nonlocal decision. We will need a preallreduce if we ever have - * multiple "good" gathervs that work on different counts for example */ - if(sendbuf != MPI_IN_PLACE && (MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS)) - pamidt = 0; - if(MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS) - pamidt = 0; - - MPIDI_Datatype_get_info(1, recvtype, contig, rsize, dt_ptr, recv_true_lb); - if(!contig) pamidt = 0; + if(sendbuf != MPI_IN_PLACE) + { + MPIDI_Datatype_get_info(sendcount, sendtype, snd_contig, + send_size, data_ptr, send_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, send_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Gatherv(sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + root, comm_ptr, mpierrno); + } + } + } + + sbuf = (char *)sendbuf + send_true_lb; + if (!snd_contig) + { + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + DLOOP_Offset last = send_size; + MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0); + MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff); + } + } + else + { + MPIDI_Datatype_get_info(1, recvtype, rcv_contig, + rcvlen, data_ptr, recv_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_GATHERV_INT, rcvlen * recvcounts[0], advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Gatherv(sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + root, comm_ptr, mpierrno); + } + } + } + } - rbuf = (char *)recvbuf + recv_true_lb; - sbuf = (void *) sendbuf; pami_xfer_t gatherv; + rbuf = (char *)recvbuf + recv_true_lb; + rcounts = (int*)recvcounts; + rdispls = (int*)displs; if(rank == root) { + if(MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS) + { + MPIDI_Datatype_get_info(1, recvtype, rcv_contig, + rcvlen, data_ptr, recv_true_lb); + if (!rcv_contig) + { + rcounts = (int*)MPIU_Malloc(size); + rdispls = (int*)MPIU_Malloc(size); + for(i = 0; i < size; i++) + { + rcounts[i] = rcvlen * recvcounts[i]; + rdispls[i] = rcvlen * displs[i]; + recv_size += rcounts[i]; + rcvcount += recvcounts[i]; + } + + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + rtype = PAMI_TYPE_BYTE; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + } + } if(sendbuf == MPI_IN_PLACE) { - sbuf = PAMI_IN_PLACE; + gatherv.cmd.xfer_gatherv_int.sndbuf = PAMI_IN_PLACE; } else { - MPIDI_Datatype_get_info(1, sendtype, contig, ssize, dt_ptr, send_true_lb); - if(!contig) pamidt = 0; - sbuf = (char *)sbuf + send_true_lb; + gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf; } - gatherv.cmd.xfer_gatherv_int.stype = stype;/* stype is ignored when sndbuf == PAMI_IN_PLACE */ - gatherv.cmd.xfer_gatherv_int.stypecount = sendcount; + gatherv.cmd.xfer_gatherv_int.stype = PAMI_TYPE_BYTE;/* stype is ignored when sndbuf == PAMI_IN_PLACE */ + gatherv.cmd.xfer_gatherv_int.stypecount = send_size; } else { - gatherv.cmd.xfer_gatherv_int.stype = stype; - gatherv.cmd.xfer_gatherv_int.stypecount = sendcount; + gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf; + gatherv.cmd.xfer_gatherv_int.stype = PAMI_TYPE_BYTE; + gatherv.cmd.xfer_gatherv_int.stypecount = send_size; } - if(pamidt == 0) - { - TRACE_ERR("GATHERV using MPICH\n"); - MPIDI_Update_last_algorithm(comm_ptr, "GATHERV_MPICH"); - return MPIR_Gatherv(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, - root, comm_ptr, mpierrno); - } gatherv.cb_done = cb_gatherv; gatherv.cookie = (void *)&gatherv_active; gatherv.cmd.xfer_gatherv_int.root = MPID_VCR_GET_LPID(comm_ptr->vcr, root); gatherv.cmd.xfer_gatherv_int.rcvbuf = rbuf; gatherv.cmd.xfer_gatherv_int.rtype = rtype; - gatherv.cmd.xfer_gatherv_int.rtypecounts = (int *) recvcounts; - gatherv.cmd.xfer_gatherv_int.rdispls = (int *) displs; + gatherv.cmd.xfer_gatherv_int.rtypecounts = (int *) rcounts; + gatherv.cmd.xfer_gatherv_int.rdispls = (int *) rdispls; - gatherv.cmd.xfer_gatherv_int.sndbuf = sbuf; const pami_metadata_t *my_gatherv_md; @@ -324,6 +391,20 @@ int MPIDO_Gatherv_simple(const void *sendbuf, TRACE_ERR("Waiting on active %d\n", gatherv_active); MPID_PROGRESS_WAIT_WHILE(gatherv_active); + if(!rcv_contig) + { + MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, + recvbuf, rcvcount, recvtype); + MPIU_Free(rcv_noncontig_buff); + if(rank == root) + { + MPIU_Free(rcounts); + MPIU_Free(rdispls); + } + } + if(!snd_contig) MPIU_Free(snd_noncontig_buff); + + TRACE_ERR("Leaving MPIDO_Gatherv_optimized\n"); return MPI_SUCCESS; } @@ -343,7 +424,10 @@ MPIDO_CSWrapper_gatherv(pami_xfer_t *gatherv, NULL); if(rc == -1) return rc; - rc = MPIDI_Dtpami_to_dtmpi( gatherv->cmd.xfer_gatherv_int.rtype, + if(gatherv->cmd.xfer_gatherv_int.rtype == PAMI_TYPE_NULL) + recvtype = MPI_DATATYPE_NULL; + else + rc = MPIDI_Dtpami_to_dtmpi( gatherv->cmd.xfer_gatherv_int.rtype, &recvtype, NULL, NULL); diff --git a/src/mpid/pamid/src/coll/reduce/mpido_reduce.c b/src/mpid/pamid/src/coll/reduce/mpido_reduce.c index 143ec74..c032354 100644 --- a/src/mpid/pamid/src/coll/reduce/mpido_reduce.c +++ b/src/mpid/pamid/src/coll/reduce/mpido_reduce.c @@ -259,13 +259,26 @@ int MPIDO_Reduce_simple(const void *sendbuf, const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); + MPIDI_Datatype_get_info(count, datatype, dt_contig, tsize, dt_null, true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_REDUCE, tsize, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, mpierrno); + } + } + } + rc = MPIDI_Datatype_to_pami(datatype, &pdt, op, &pop, &mu); pami_xfer_t reduce; const pami_metadata_t *my_reduce_md=NULL; volatile unsigned reduce_active = 1; - MPIDI_Datatype_get_info(count, datatype, dt_contig, tsize, dt_null, true_lb); if(rc != MPI_SUCCESS || !dt_contig) { return MPIR_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, mpierrno); diff --git a/src/mpid/pamid/src/coll/scan/mpido_scan.c b/src/mpid/pamid/src/coll/scan/mpido_scan.c index b9e848e..26d9fc5 100644 --- a/src/mpid/pamid/src/coll/scan/mpido_scan.c +++ b/src/mpid/pamid/src/coll/scan/mpido_scan.c @@ -228,12 +228,27 @@ int MPIDO_Doscan_simple(const void *sendbuf, void *recvbuf, const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); - rc = MPIDI_Datatype_to_pami(datatype, &pdt, op, &pop, &mu); - pami_xfer_t scan; volatile unsigned scan_active = 1; MPIDI_Datatype_get_info(count, datatype, dt_contig, tsize, dt_null, true_lb); - + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCAN, tsize, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + if(exflag) + return MPIR_Exscan(sendbuf, recvbuf, count, datatype, op, comm_ptr, mpierrno); + else + return MPIR_Scan(sendbuf, recvbuf, count, datatype, op, comm_ptr, mpierrno); + } + } + } + + rc = MPIDI_Datatype_to_pami(datatype, &pdt, op, &pop, &mu); + if(rc != MPI_SUCCESS || !dt_contig) { if(exflag) diff --git a/src/mpid/pamid/src/coll/scatter/mpido_scatter.c b/src/mpid/pamid/src/coll/scatter/mpido_scatter.c index e5aad3b..a511f96 100644 --- a/src/mpid/pamid/src/coll/scatter/mpido_scatter.c +++ b/src/mpid/pamid/src/coll/scatter/mpido_scatter.c @@ -324,39 +324,98 @@ int MPIDO_Scatter_simple(const void *sendbuf, int *mpierrno) { MPID_Datatype * data_ptr; - MPI_Aint true_lb = 0; - int contig, nbytes = 0; const int rank = comm_ptr->rank; - int success = 1; - pami_type_t stype, rtype = NULL; + int success = 1, snd_contig = 1, rcv_contig = 1; + void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; + void *sbuf = NULL, *rbuf = NULL; + size_t send_size = 0; + size_t recv_size = 0; + MPI_Aint snd_true_lb = 0, rcv_true_lb = 0; int tmp; int use_pami = 1; + MPID_Segment segment; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); - if(MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS) - use_pami = 0; - if(recvbuf != MPI_IN_PLACE && (MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS)) - use_pami = 0; + if (rank == root && sendtype != MPI_DATATYPE_NULL && sendcount >= 0) + { + MPIDI_Datatype_get_info(sendcount, sendtype, snd_contig, + send_size, data_ptr, snd_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTER, send_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Scatter(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, + root, comm_ptr, mpierrno); + } + } + } + } + + if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0) + { + MPIDI_Datatype_get_info(recvcount, recvtype, rcv_contig, + recv_size, data_ptr, rcv_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTER, recv_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Scatter(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, + root, comm_ptr, mpierrno); + } + } + } + } + sbuf = (char *)sendbuf + snd_true_lb; + rbuf = (char *)recvbuf + rcv_true_lb; if (rank == root) { - if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0)/* Should this be send or recv? */ + if (send_size) { - MPIDI_Datatype_get_info(sendcount, sendtype, contig, - nbytes, data_ptr, true_lb); - if (!contig) success = 0; + sbuf = (char *)sendbuf + snd_true_lb; + if (!snd_contig) + { + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + DLOOP_Offset last = send_size; + MPID_Segment_init(sendbuf, sendcount, sendtype, &segment, 0); + MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff); + } } else success = 0; - if (success) + if (success && recvbuf != MPI_IN_PLACE) { - if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0) + if (recv_size) { - MPIDI_Datatype_get_info(recvcount, recvtype, contig, - nbytes, data_ptr, true_lb); - if (!contig) success = 0; + rbuf = (char *)recvbuf + rcv_true_lb; + if (!rcv_contig) + { + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + } } else success = 0; } @@ -364,17 +423,25 @@ int MPIDO_Scatter_simple(const void *sendbuf, else { - if (sendtype != MPI_DATATYPE_NULL && sendcount >= 0)/* Should this be send or recv? */ + if (recv_size)/* Should this be send or recv? */ { - MPIDI_Datatype_get_info(recvcount, recvtype, contig, - nbytes, data_ptr, true_lb); - if (!contig) success = 0; + rbuf = (char *)recvbuf + rcv_true_lb; + if (!rcv_contig) + { + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + } } else success = 0; } - if(!use_pami || !success) + if(!success) { MPIDI_Update_last_algorithm(comm_ptr, "SCATTER_MPICH"); return MPIR_Scatter(sendbuf, sendcount, sendtype, @@ -394,12 +461,12 @@ int MPIDO_Scatter_simple(const void *sendbuf, scatter.cb_done = cb_scatter; scatter.cookie = (void *)&scatter_active; scatter.cmd.xfer_scatter.root = MPID_VCR_GET_LPID(comm_ptr->vcr, root); - scatter.cmd.xfer_scatter.sndbuf = (void *)sendbuf; - scatter.cmd.xfer_scatter.stype = stype; - scatter.cmd.xfer_scatter.stypecount = sendcount; - scatter.cmd.xfer_scatter.rcvbuf = (void *)recvbuf; - scatter.cmd.xfer_scatter.rtype = rtype;/* rtype is ignored when rcvbuf == PAMI_IN_PLACE */ - scatter.cmd.xfer_scatter.rtypecount = recvcount; + scatter.cmd.xfer_scatter.sndbuf = (void *)sbuf; + scatter.cmd.xfer_scatter.stype = PAMI_TYPE_BYTE; + scatter.cmd.xfer_scatter.stypecount = send_size; + scatter.cmd.xfer_scatter.rcvbuf = (void *)rbuf; + scatter.cmd.xfer_scatter.rtype = PAMI_TYPE_BYTE;/* rtype is ignored when rcvbuf == PAMI_IN_PLACE */ + scatter.cmd.xfer_scatter.rtypecount = recv_size; if(recvbuf == MPI_IN_PLACE) { @@ -413,6 +480,14 @@ int MPIDO_Scatter_simple(const void *sendbuf, TRACE_ERR("Waiting on active %d\n", scatter_active); MPID_PROGRESS_WAIT_WHILE(scatter_active); + if(!rcv_contig) + { + MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, + recvbuf, recvcount, recvtype); + MPIU_Free(rcv_noncontig_buff); + } + if(!snd_contig) MPIU_Free(snd_noncontig_buff); + TRACE_ERR("Leaving MPIDO_Scatter_optimized\n"); diff --git a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c index 51b01ec..f868c00 100644 --- a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c +++ b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c @@ -446,52 +446,115 @@ int MPIDO_Scatterv_simple(const void *sendbuf, { int snd_contig = 1; int rcv_contig = 1; - int tmp, pamidt = 1; - int ssize, rsize; + int send_size = 0, recv_size = 0; + int ssize = 0; MPID_Datatype *dt_ptr = NULL; MPI_Aint send_true_lb=0, recv_true_lb=0; - char *sbuf, *rbuf; - pami_type_t stype, rtype = NULL; + void *snd_noncontig_buff = NULL, *rcv_noncontig_buff = NULL; + void *sbuf = NULL, *rbuf = NULL; + int *sdispls = NULL, *scounts = NULL; + int sndlen = 0; + int sndcount = 0; + MPID_Segment segment; + int tmp, i; + pami_type_t stype = PAMI_TYPE_NULL; const int rank = comm_ptr->rank; + const int size = comm_ptr->local_size; const struct MPIDI_Comm* const mpid = &(comm_ptr->mpid); + if (rank == root && sendtype != MPI_DATATYPE_NULL && sendcounts[0] >= 0) + { + MPIDI_Datatype_get_info(1, sendtype, snd_contig, ssize, dt_ptr, send_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, ssize * sendcounts[0], advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype, + recvbuf, recvcount, recvtype, + root, comm_ptr, mpierrno); + } + } + } + } + + if (recvtype != MPI_DATATYPE_NULL && recvcount >= 0) + { + MPIDI_Datatype_get_info(recvcount, recvtype, rcv_contig, + recv_size, dt_ptr, recv_true_lb); + if(MPIDI_Pamix_collsel_advise != NULL) + { + advisor_algorithm_t advisor_algorithms[1]; + int num_algorithms = MPIDI_Pamix_collsel_advise(mpid->collsel_fast_query, PAMI_XFER_SCATTERV_INT, recv_size, advisor_algorithms, 1); + if(num_algorithms) + { + if(advisor_algorithms[0].algorithm_type == COLLSEL_EXTERNAL_ALGO) + { + return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype, + recvbuf, recvcount, recvtype, + root, comm_ptr, mpierrno); + } + } + } + } + pami_xfer_t scatterv; const pami_metadata_t *my_scatterv_md; volatile unsigned scatterv_active = 1; - - if((recvbuf != MPI_IN_PLACE) && MPIDI_Datatype_to_pami(recvtype, &rtype, -1, NULL, &tmp) != MPI_SUCCESS) - pamidt = 0; - - if(MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS) - pamidt = 0; - MPIDI_Datatype_get_info(1, sendtype, snd_contig, ssize, dt_ptr, send_true_lb); - if(recvbuf != MPI_IN_PLACE) - MPIDI_Datatype_get_info(1, recvtype, rcv_contig, rsize, dt_ptr, recv_true_lb); - - if(pamidt == 0 || !snd_contig || !rcv_contig) + sbuf = (char *)sendbuf + send_true_lb; + rbuf = (char *)recvbuf + recv_true_lb; + scounts = (int*)sendcounts; + sdispls = (int*)displs; + if(rank == root) { - TRACE_ERR("Scatterv using MPICH\n"); - MPIDI_Update_last_algorithm(comm_ptr, "SCATTERV_MPICH"); - return MPIR_Scatterv(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, - root, comm_ptr, mpierrno); + if(MPIDI_Datatype_to_pami(sendtype, &stype, -1, NULL, &tmp) != MPI_SUCCESS) + { + if (!snd_contig) + { + scounts = (int*)MPIU_Malloc(size); + sdispls = (int*)MPIU_Malloc(size); + for(i = 0; i < size; i++) + { + scounts[i] = ssize * sendcounts[i]; + sdispls[i] = ssize * displs[i]; + send_size += scounts[i]; + sndcount += sendcounts[i]; + } + snd_noncontig_buff = MPIU_Malloc(send_size); + sbuf = snd_noncontig_buff; + stype = PAMI_TYPE_BYTE; + if(snd_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + DLOOP_Offset last = send_size; + MPID_Segment_init(sendbuf, sndcount, sendtype, &segment, 0); + MPID_Segment_pack(&segment, 0, &last, snd_noncontig_buff); + } + } + if(recvbuf == MPI_IN_PLACE) + { + rbuf = PAMI_IN_PLACE; + } } - - sbuf = (char *)sendbuf + send_true_lb; - rbuf = recvbuf; - - if(rank == root) + if(recvbuf != MPI_IN_PLACE) { - if(recvbuf == MPI_IN_PLACE) - { - rbuf = PAMI_IN_PLACE; - } - else - { - rbuf = (char *)recvbuf + recv_true_lb; - } + if (!rcv_contig) + { + rcv_noncontig_buff = MPIU_Malloc(recv_size); + rbuf = rcv_noncontig_buff; + if(rcv_noncontig_buff == NULL) + { + MPID_Abort(NULL, MPI_ERR_NO_SPACE, 1, + "Fatal: Cannot allocate pack buffer"); + } + } } scatterv.cb_done = cb_scatterv; @@ -504,10 +567,10 @@ int MPIDO_Scatterv_simple(const void *sendbuf, scatterv.cmd.xfer_scatterv_int.rcvbuf = rbuf; scatterv.cmd.xfer_scatterv_int.sndbuf = sbuf; scatterv.cmd.xfer_scatterv_int.stype = stype; - scatterv.cmd.xfer_scatterv_int.rtype = rtype;/* rtype is ignored when rcvbuf == PAMI_IN_PLACE */ - scatterv.cmd.xfer_scatterv_int.stypecounts = (int *) sendcounts; - scatterv.cmd.xfer_scatterv_int.rtypecount = recvcount; - scatterv.cmd.xfer_scatterv_int.sdispls = (int *) displs; + scatterv.cmd.xfer_scatterv_int.rtype = PAMI_TYPE_BYTE;/* rtype is ignored when rcvbuf == PAMI_IN_PLACE */ + scatterv.cmd.xfer_scatterv_int.stypecounts = (int *) scounts; + scatterv.cmd.xfer_scatterv_int.rtypecount = recv_size; + scatterv.cmd.xfer_scatterv_int.sdispls = (int *) sdispls; MPIDI_Update_last_algorithm(comm_ptr, my_scatterv_md->name); @@ -521,6 +584,19 @@ int MPIDO_Scatterv_simple(const void *sendbuf, TRACE_ERR("Waiting on active %d\n", scatterv_active); MPID_PROGRESS_WAIT_WHILE(scatterv_active); + if(!rcv_contig) + { + MPIR_Localcopy(rcv_noncontig_buff, recv_size, MPI_CHAR, + recvbuf, recvcount, recvtype); + MPIU_Free(rcv_noncontig_buff); + } + if(!snd_contig) + { + MPIU_Free(snd_noncontig_buff); + MPIU_Free(scounts); + MPIU_Free(sdispls); + } + TRACE_ERR("Leaving MPIDO_Scatterv_optimized\n"); return MPI_SUCCESS; } @@ -530,12 +606,17 @@ int MPIDO_CSWrapper_scatterv(pami_xfer_t *scatterv, void *comm) { - int mpierrno = 0; + int mpierrno = 0, rc = 0; MPID_Comm *comm_ptr = (MPID_Comm*)comm; MPI_Datatype sendtype, recvtype; void *rbuf; MPIDI_coll_check_in_place(scatterv->cmd.xfer_scatterv_int.rcvbuf, &rbuf); - int rc = MPIDI_Dtpami_to_dtmpi( scatterv->cmd.xfer_scatterv_int.stype, + /* Since collective selection in PAMI fixes message size when selecting the + algorithm, this wrapper function may be called back from PAMI */ + if(PAMI_TYPE_NULL == scatterv->cmd.xfer_scatterv_int.stype) + sendtype = MPI_DATATYPE_NULL; + else + rc = MPIDI_Dtpami_to_dtmpi( scatterv->cmd.xfer_scatterv_int.stype, &sendtype, NULL, NULL); diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index 844f0cd..1de14a0 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -391,6 +391,19 @@ void MPIDI_Coll_comm_destroy(MPID_Comm *comm) MPIU_TestFree(&comm->mpid.coll_metadata[i][1]); } + + if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE) + { + /* Destroy the fast query object. */ + pami_extension_collsel_query_destroy pamix_collsel_query_destroy = + (pami_extension_collsel_query_destroy) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_query_destroy"); + if(pamix_collsel_query_destroy != NULL) + { + pamix_collsel_query_destroy(&(comm->mpid.collsel_fast_query)); + } + } + TRACE_ERR("Destroying geometry\n"); geom_destroy_post.client = MPIDI_Client; diff --git a/src/mpid/pamid/src/comm/mpid_selectcolls.c b/src/mpid/pamid/src/comm/mpid_selectcolls.c index 834542f..886dbd8 100644 --- a/src/mpid/pamid/src/comm/mpid_selectcolls.c +++ b/src/mpid/pamid/src/comm/mpid_selectcolls.c @@ -35,7 +35,7 @@ external_algorithm_t ext_algorithm; ext_algorithms[0].algorithm.external = ext_algorithm; \ ext_algorithms[0].metadata = &ext_metadata; \ ext_algorithms[0].algorithm_type = COLLSEL_EXTERNAL_ALGO; \ - collsel_register_algorithms(MPIDI_Collsel_advisor_table, \ + pamix_collsel_register_algorithms(MPIDI_Collsel_advisor_table, \ comm->mpid.geometry, \ xfer_type, \ &ext_algorithms[0], \ @@ -481,10 +481,23 @@ void MPIDI_Comm_coll_envvars(MPID_Comm *comm) it, then use auto coll sel.. Otherwise, go through the manual coll sel code path. */ if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE) { - pami_extension_collsel_register_algorithms collsel_register_algorithms = + /* Create a fast query object, cache it on the comm/geometry and use it in each collective */ + pami_extension_collsel_query_create pamix_collsel_query_create = + (pami_extension_collsel_query_create) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_query_create"); + if(pamix_collsel_query_create != NULL) + { + pamix_collsel_query_create(MPIDI_Collsel_advisor_table, comm->mpid.geometry, &(comm->mpid.collsel_fast_query)); + } + + MPIDI_Pamix_collsel_advise = /* Get the function pointer and cache it */ + (pami_extension_collsel_advise) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_advise"); + + pami_extension_collsel_register_algorithms pamix_collsel_register_algorithms = (pami_extension_collsel_register_algorithms) PAMI_Extension_symbol(MPIDI_Collsel_extension, "Collsel_register_algorithms"); - if(collsel_register_algorithms != NULL) + if(pamix_collsel_register_algorithms != NULL) { /* ************ Barrier ************ */ diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 5fbba71..022a2fd 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -282,6 +282,7 @@ advisor_t MPIDI_Collsel_advisor; advisor_table_t MPIDI_Collsel_advisor_table; advisor_params_t MPIDI_Collsel_advisor_params; char *MPIDI_Collsel_output_file; +pami_extension_collsel_advise MPIDI_Pamix_collsel_advise; static void MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char **world_tasks) { http://git.mpich.org/mpich.git/commitdiff/f4f0de65736a8abf6c96b727e87b08299… commit f4f0de65736a8abf6c96b727e87b082992616cf5 Author: sssharka <sssharka(a)us.ibm.com> Date: Fri Jan 25 21:07:57 2013 -0500 External algorithm registration for collective selection (ibm) F183421 (ibm) 6463405fd65ec7a45b137334c7608d30f8c29055 Signed-off-by: Bob Cernohous <bobc(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_constants.h b/src/mpid/pamid/include/mpidi_constants.h index be0d18c..a16292e 100644 --- a/src/mpid/pamid/include/mpidi_constants.h +++ b/src/mpid/pamid/include/mpidi_constants.h @@ -113,6 +113,7 @@ enum MPID_AUTO_SELECT_COLLS_SCAN = ((int)((MPID_AUTO_SELECT_COLLS_REDUCE << 1) & 0xFFFFFFFF)), MPID_AUTO_SELECT_COLLS_SCATTER = ((int)((MPID_AUTO_SELECT_COLLS_SCAN << 1) & 0xFFFFFFFF)), MPID_AUTO_SELECT_COLLS_SCATTERV = ((int)((MPID_AUTO_SELECT_COLLS_SCATTER << 1) & 0xFFFFFFFF)), + MPID_AUTO_SELECT_COLLS_TUNE = 0x80000000, MPID_AUTO_SELECT_COLLS_ALL = 0xFFFFFFFF, }; diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 3dd8b9a..dc87146 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -442,5 +442,119 @@ struct MPIDI_Win } sync; }; +/** + * \brief Structures and typedefs for collective selection extensions in PAMI + */ + +typedef void* advisor_t; +typedef void* advisor_table_t; +typedef void* advisor_attribute_name_t; + +typedef union +{ + size_t intval; + double doubleval; + const char * chararray; + const size_t * intarray; +} advisor_attribute_value_t; + +typedef struct +{ + advisor_attribute_name_t name; + advisor_attribute_value_t value; +} advisor_configuration_t; + +typedef struct { + pami_xfer_type_t *collectives; + size_t num_collectives; + size_t *procs_per_node; + size_t num_procs_per_node; + size_t *geometry_sizes; + size_t num_geometry_sizes; + size_t *message_sizes; + size_t num_message_sizes; + int iter; + int verify; + int verbose; + int checkpoint; +} advisor_params_t; + +typedef enum +{ + COLLSEL_ALGO = 0, /* 'Always works' PAMI algorithm */ + COLLSEL_QUERY_ALGO, /* 'Must query' PAMI algorithm */ + COLLSEL_EXTERNAL_ALGO, /* External algorithm */ +} advisor_algorithm_type_t; + +/* External algorithm callback function */ +typedef pami_result_t (*external_algorithm_fn)(pami_xfer_t *, void *); + +typedef struct +{ + external_algorithm_fn callback; + void *cookie; +} external_algorithm_t; + +typedef struct +{ + union + { + pami_algorithm_t internal;/* PAMI Algorithm */ + external_algorithm_t external;/* External Algorithm */ + } algorithm; + pami_metadata_t *metadata; + advisor_algorithm_type_t algorithm_type; +} advisor_algorithm_t; + + +typedef pami_result_t (*pami_extension_collsel_init) (pami_client_t, + advisor_configuration_t [], + size_t, + pami_context_t [], + size_t, + advisor_t *); + +typedef pami_result_t (*pami_extension_collsel_destroy) (advisor_t *); + +typedef int (*pami_extension_collsel_initialized) (pami_client_t, advisor_t *); + +typedef pami_result_t (*pami_extension_collsel_table_load) (advisor_t, + char *, + advisor_table_t *); + +typedef pami_result_t (*pami_extension_collsel_get_collectives) (advisor_table_t, + pami_xfer_type_t **, + unsigned *); + +typedef pami_result_t (*pami_extension_collsel_register_algorithms) (advisor_table_t, + pami_geometry_t, + pami_xfer_type_t, + advisor_algorithm_t *, + size_t); + +typedef pami_result_t (*external_geometry_create_fn)(pami_geometry_range_t* task_slices, + size_t slice_count, + pami_geometry_t *geometry, + void **cookie); + +typedef pami_result_t (*external_geometry_destroy_fn)(void *cookie); + +typedef pami_result_t (*external_register_algorithms_fn)(void *cookie, + pami_xfer_type_t collective, + advisor_algorithm_t **algorithms, + size_t *num_algorithms); + +typedef struct +{ + external_geometry_create_fn geometry_create; + external_geometry_destroy_fn geometry_destroy; + external_register_algorithms_fn register_algorithms; +} external_geometry_ops_t; + +typedef pami_result_t (*pami_extension_collsel_table_generate) (advisor_t, + char *, + advisor_params_t *, + external_geometry_ops_t *, + int); #endif diff --git a/src/mpid/pamid/include/mpidi_externs.h b/src/mpid/pamid/include/mpidi_externs.h index cb64f00..f9a9683 100644 --- a/src/mpid/pamid/include/mpidi_externs.h +++ b/src/mpid/pamid/include/mpidi_externs.h @@ -34,5 +34,10 @@ extern pami_context_t MPIDI_Context[]; extern MPIDI_Process_t MPIDI_Process; +extern advisor_table_t MPIDI_Collsel_advisor_table; +extern pami_extension_t MPIDI_Collsel_extension; +extern advisor_params_t MPIDI_Collsel_advisor_params; +extern char *MPIDI_Collsel_output_file; + #endif diff --git a/src/mpid/pamid/include/mpidi_prototypes.h b/src/mpid/pamid/include/mpidi_prototypes.h index e40baa3..55566f2 100644 --- a/src/mpid/pamid/include/mpidi_prototypes.h +++ b/src/mpid/pamid/include/mpidi_prototypes.h @@ -217,6 +217,17 @@ void MPIDI_Coll_comm_destroy (MPID_Comm *comm); void MPIDI_Env_setup (); void MPIDI_Comm_world_setup (); +pami_result_t MPIDI_Comm_create_from_pami_geom(pami_geometry_range_t *task_slices, + size_t slice_count, + pami_geometry_t *geometry, + void **cookie); +pami_result_t MPIDI_Comm_destroy_external(void *comm_ext); +pami_result_t MPIDI_Register_algorithms_ext(void *cookie, + pami_xfer_type_t collective, + advisor_algorithm_t **algorithms, + size_t *num_algorithms); +int MPIDI_collsel_pami_tune_parse_params(int argc, char ** argv); +void MPIDI_collsel_pami_tune_cleanup(); void MPIDI_Coll_Comm_create (MPID_Comm *comm); void MPIDI_Coll_Comm_destroy(MPID_Comm *comm); void MPIDI_Comm_coll_query (MPID_Comm *comm); @@ -226,23 +237,28 @@ void MPIDI_Coll_register (void); int MPIDO_Bcast(void *buffer, int count, MPI_Datatype dt, int root, MPID_Comm *comm_ptr, int *mpierrno); int MPIDO_Bcast_simple(void *buffer, int count, MPI_Datatype dt, int root, MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_bcast(pami_xfer_t *bcast, void *comm); int MPIDO_Barrier(MPID_Comm *comm_ptr, int *mpierrno); int MPIDO_Barrier_simple(MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_barrier(pami_xfer_t *barrier, void *comm); int MPIDO_Allreduce(const void *sbuffer, void *rbuffer, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr, int *mpierrno); int MPIDO_Allreduce_simple(const void *sbuffer, void *rbuffer, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_allreduce(pami_xfer_t *allreduce, void *comm); int MPIDO_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPID_Comm *comm_ptr, int *mpierrno); int MPIDO_Reduce_simple(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_reduce(pami_xfer_t *reduce, void *comm); int MPIDO_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPID_Comm *comm_ptr, int *mpierrno); int MPIDO_Allgather_simple(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_allgather(pami_xfer_t *allgather, void *comm); int MPIDO_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int *recvcounts, const int *displs, @@ -250,6 +266,7 @@ int MPIDO_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int MPIDO_Allgatherv_simple(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, MPID_Comm * comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_allgatherv(pami_xfer_t *allgatherv, void *comm); int MPIDO_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, MPID_Comm * comm_ptr, @@ -261,6 +278,7 @@ int MPIDO_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int MPIDO_Gather_simple(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPID_Comm * comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_gather(pami_xfer_t *gather, void *comm); int MPIDO_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, @@ -268,11 +286,13 @@ int MPIDO_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int MPIDO_Gatherv_simple(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, int root, MPID_Comm * comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_gatherv(pami_xfer_t *gatherv, void *comm); int MPIDO_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm * comm_ptr, int *mpierrno); int MPIDO_Scan_simple(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm * comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_scan(pami_xfer_t *scan, void *comm); int MPIDO_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm * comm_ptr, int *mpierrno); @@ -285,6 +305,7 @@ int MPIDO_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int MPIDO_Scatter_simple(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPID_Comm * comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_scatter(pami_xfer_t *scatter, void *comm); int MPIDO_Scatterv(const void *sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, @@ -294,6 +315,7 @@ int MPIDO_Scatterv_simple(const void *sendbuf, const int *sendcounts, const int MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPID_Comm * comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_scatterv(pami_xfer_t *scatterv, void *comm); int MPIDO_Alltoallv(const void *sendbuf, const int *sendcounts, const int *senddispls, MPI_Datatype sendtype, @@ -305,6 +327,7 @@ int MPIDO_Alltoallv_simple(const void *sendbuf, const int *sendcounts, const int void *recvbuf, const int *recvcounts, const int *recvdispls, MPI_Datatype recvtype, MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_alltoallv(pami_xfer_t *alltoallv, void *comm); int MPIDO_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, @@ -312,12 +335,18 @@ int MPIDO_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int MPIDO_Alltoall_simple(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPID_Comm *comm_ptr, int *mpierrno); +int MPIDO_CSWrapper_alltoall(pami_xfer_t *alltoall, void *comm); int MPIDI_Datatype_to_pami(MPI_Datatype dt, pami_type_t *pdt, MPI_Op op, pami_data_function *pop, int *musupport); + +int MPIDI_Dtpami_to_dtmpi(pami_type_t pdt, + MPI_Datatype *dt, + pami_data_function pop, + MPI_Op *op); void MPIDI_Op_to_string(MPI_Op op, char *string); pami_result_t MPIDI_Pami_post_wrapper(pami_context_t context, void *cookie); diff --git a/src/mpid/pamid/include/mpidimpl.h b/src/mpid/pamid/include/mpidimpl.h index 8e5d24f..aef965a 100644 --- a/src/mpid/pamid/include/mpidimpl.h +++ b/src/mpid/pamid/include/mpidimpl.h @@ -29,6 +29,20 @@ #include <mpix.h> + +static inline void MPIDI_coll_check_in_place(void* src, void** dst) +{ + if(MPI_IN_PLACE == PAMI_IN_PLACE) + *dst = src; + else + { + if(src == PAMI_IN_PLACE) + *dst = MPI_IN_PLACE; + else + *dst = src; + } +} + #ifdef DYNAMIC_TASKING #define MPIDI_MAX_KVS_VALUE_LEN 4096 diff --git a/src/mpid/pamid/src/coll/allgather/mpido_allgather.c b/src/mpid/pamid/src/coll/allgather/mpido_allgather.c index a63c91f..bb9ecf3 100644 --- a/src/mpid/pamid/src/coll/allgather/mpido_allgather.c +++ b/src/mpid/pamid/src/coll/allgather/mpido_allgather.c @@ -685,3 +685,37 @@ MPIDO_Allgather_simple(const void *sendbuf, TRACE_ERR("Allgather done\n"); return MPI_SUCCESS; } + + +int +MPIDO_CSWrapper_allgather(pami_xfer_t *allgather, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *sbuf; + MPIDI_coll_check_in_place(allgather->cmd.xfer_allgather.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( allgather->cmd.xfer_allgather.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( allgather->cmd.xfer_allgather.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Allgather(sbuf, + allgather->cmd.xfer_allgather.stypecount, sendtype, + allgather->cmd.xfer_allgather.rcvbuf, + allgather->cmd.xfer_allgather.rtypecount, recvtype, + comm_ptr, &mpierrno); + if(allgather->cb_done && rc == 0) + allgather->cb_done(NULL, allgather->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c b/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c index b01f4db..fbf826f 100644 --- a/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c +++ b/src/mpid/pamid/src/coll/allgatherv/mpido_allgatherv.c @@ -705,3 +705,39 @@ MPIDO_Allgatherv_simple(const void *sendbuf, return MPI_SUCCESS; } + + +int +MPIDO_CSWrapper_allgatherv(pami_xfer_t *allgatherv, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *sbuf; + MPIDI_coll_check_in_place(allgatherv->cmd.xfer_allgatherv_int.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( allgatherv->cmd.xfer_allgatherv_int.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( allgatherv->cmd.xfer_allgatherv_int.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Allgatherv(sbuf, + allgatherv->cmd.xfer_allgatherv_int.stypecount, sendtype, + allgatherv->cmd.xfer_allgatherv_int.rcvbuf, + allgatherv->cmd.xfer_allgatherv_int.rtypecounts, + allgatherv->cmd.xfer_allgatherv_int.rdispls, recvtype, + comm_ptr, &mpierrno); + if(allgatherv->cb_done && rc == 0) + allgatherv->cb_done(NULL, allgatherv->cookie, PAMI_SUCCESS); + return rc; + + +} + diff --git a/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c b/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c index 9edab4e..ff7ca70 100644 --- a/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c +++ b/src/mpid/pamid/src/coll/allreduce/mpido_allreduce.c @@ -465,3 +465,30 @@ int MPIDO_Allreduce_simple(const void *sendbuf, TRACE_ERR("Allreduce done\n"); return MPI_SUCCESS; } + +int +MPIDO_CSWrapper_allreduce(pami_xfer_t *allreduce, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype type; + MPI_Op op; + void *sbuf; + MPIDI_coll_check_in_place(allreduce->cmd.xfer_allreduce.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( allreduce->cmd.xfer_allreduce.stype, + &type, + allreduce->cmd.xfer_allreduce.op, + &op); + if(rc == -1) return rc; + + rc = MPIR_Allreduce(sbuf, + allreduce->cmd.xfer_allreduce.rcvbuf, + allreduce->cmd.xfer_allreduce.rtypecount, + type, op, comm_ptr, &mpierrno); + if(allreduce->cb_done && rc == 0) + allreduce->cb_done(NULL, allreduce->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c b/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c index 5b33afb..ba4f2a0 100644 --- a/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c +++ b/src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c @@ -290,3 +290,37 @@ int MPIDO_Alltoall_simple(const void *sendbuf, TRACE_ERR("Leaving MPIDO_Alltoall_optimized\n"); return MPI_SUCCESS; } + + +int +MPIDO_CSWrapper_alltoall(pami_xfer_t *alltoall, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *sbuf; + MPIDI_coll_check_in_place(alltoall->cmd.xfer_alltoall.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( alltoall->cmd.xfer_alltoall.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( alltoall->cmd.xfer_alltoall.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Alltoall_intra(sbuf, + alltoall->cmd.xfer_alltoall.stypecount, sendtype, + alltoall->cmd.xfer_alltoall.rcvbuf, + alltoall->cmd.xfer_alltoall.rtypecount, recvtype, + comm_ptr, &mpierrno); + if(alltoall->cb_done && rc == 0) + alltoall->cb_done(NULL, alltoall->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c index 87a6df1..e48ec66 100644 --- a/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c +++ b/src/mpid/pamid/src/coll/alltoallv/mpido_alltoallv.c @@ -297,3 +297,38 @@ int MPIDO_Alltoallv_simple(const void *sendbuf, return MPI_SUCCESS; } + +int +MPIDO_CSWrapper_alltoallv(pami_xfer_t *alltoallv, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *sbuf; + MPIDI_coll_check_in_place(alltoallv->cmd.xfer_alltoallv_int.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( alltoallv->cmd.xfer_alltoallv_int.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( alltoallv->cmd.xfer_alltoallv_int.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Alltoallv(sbuf, + alltoallv->cmd.xfer_alltoallv_int.stypecounts, + alltoallv->cmd.xfer_alltoallv_int.sdispls, sendtype, + alltoallv->cmd.xfer_alltoallv_int.rcvbuf, + alltoallv->cmd.xfer_alltoallv_int.rtypecounts, + alltoallv->cmd.xfer_alltoallv_int.rdispls, recvtype, + comm_ptr, &mpierrno); + if(alltoallv->cb_done && rc == 0) + alltoallv->cb_done(NULL, alltoallv->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/barrier/mpido_barrier.c b/src/mpid/pamid/src/coll/barrier/mpido_barrier.c index 23e5e6d..e4df53d 100644 --- a/src/mpid/pamid/src/coll/barrier/mpido_barrier.c +++ b/src/mpid/pamid/src/coll/barrier/mpido_barrier.c @@ -130,3 +130,17 @@ int MPIDO_Barrier_simple(MPID_Comm *comm_ptr, int *mpierrno) TRACE_ERR("Exiting MPIDO_Barrier_optimized\n"); return 0; } + +int +MPIDO_CSWrapper_barrier(pami_xfer_t *barrier, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + int rc = MPIR_Barrier(comm_ptr, &mpierrno); + if(barrier->cb_done && rc == 0) + barrier->cb_done(NULL, barrier->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/bcast/mpido_bcast.c b/src/mpid/pamid/src/coll/bcast/mpido_bcast.c index e86696e..45450fb 100644 --- a/src/mpid/pamid/src/coll/bcast/mpido_bcast.c +++ b/src/mpid/pamid/src/coll/bcast/mpido_bcast.c @@ -346,3 +346,26 @@ int MPIDO_Bcast_simple(void *buffer, TRACE_ERR("Exiting MPIDO_Bcast_optimized\n"); return 0; } + + +int +MPIDO_CSWrapper_bcast(pami_xfer_t *bcast, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype type; + int rc = MPIDI_Dtpami_to_dtmpi( bcast->cmd.xfer_broadcast.type, + &type, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Bcast_intra(bcast->cmd.xfer_broadcast.buf, + bcast->cmd.xfer_broadcast.typecount, type, + bcast->cmd.xfer_broadcast.root, comm_ptr, &mpierrno); + if(bcast->cb_done && rc == 0) + bcast->cb_done(NULL, bcast->cookie, PAMI_SUCCESS); + return rc; +} + diff --git a/src/mpid/pamid/src/coll/coll_utils.c b/src/mpid/pamid/src/coll/coll_utils.c index 582c922..fcbbf95 100644 --- a/src/mpid/pamid/src/coll/coll_utils.c +++ b/src/mpid/pamid/src/coll/coll_utils.c @@ -72,6 +72,11 @@ pami_result_t MPIDI_Pami_post_wrapper(pami_context_t context, void *cookie) (x) == MPI_DOUBLE_INT || (x) == MPI_LONG_INT || \ (x) == MPI_2INT || (x) == MPI_SHORT_INT || \ (x) == MPI_LONG_DOUBLE_INT ) + +#define isPAMI_LOC_TYPE(x) ( (x) == PAMI_TYPE_LOC_2INT || (x) == PAMI_TYPE_LOC_FLOAT_INT || \ + (x) == PAMI_TYPE_LOC_DOUBLE_INT || (x) == PAMI_TYPE_LOC_SHORT_INT || \ + (x) == PAMI_TYPE_LOC_LONG_INT || (x) == PAMI_TYPE_LOC_LONGDOUBLE_INT || \ + (x) == PAMI_TYPE_LOC_2FLOAT || (x) == PAMI_TYPE_LOC_2DOUBLE ) #define isBOOL(x) ( (x) == MPI_C_BOOL ) @@ -263,3 +268,132 @@ int MPIDI_Datatype_to_pami(MPI_Datatype dt, return MPI_SUCCESS; } + +int MPIDI_Dtpami_to_dtmpi( pami_type_t pdt, + MPI_Datatype *dt, + pami_data_function pop, + MPI_Op *op) +{ + *dt = MPI_DATATYPE_NULL; + if(pop != NULL) + *op = MPI_OP_NULL; + + if(pdt == PAMI_TYPE_SIGNED_INT) *dt = MPI_INT; + else if(pdt == PAMI_TYPE_UNSIGNED_INT) *dt = MPI_UNSIGNED; + else if(pdt == PAMI_TYPE_FLOAT) *dt = MPI_FLOAT; + else if(pdt == PAMI_TYPE_DOUBLE) *dt = MPI_DOUBLE; + else if(pdt == PAMI_TYPE_LONG_DOUBLE) *dt = MPI_LONG_DOUBLE; + else if(pdt == PAMI_TYPE_SIGNED_CHAR) *dt = MPI_CHAR; + else if(pdt == PAMI_TYPE_UNSIGNED_CHAR) *dt = MPI_UNSIGNED_CHAR; + else if(pdt == PAMI_TYPE_BYTE) *dt = MPI_BYTE; + else if(pdt == PAMI_TYPE_SIGNED_SHORT) *dt = MPI_SHORT; + else if(pdt == PAMI_TYPE_UNSIGNED_SHORT) *dt = MPI_UNSIGNED_SHORT; + else if(pdt == PAMI_TYPE_SIGNED_LONG_LONG) *dt = MPI_LONG_LONG; + else if(pdt == PAMI_TYPE_UNSIGNED_LONG_LONG) *dt = MPI_UNSIGNED_LONG_LONG; + else if(pdt == PAMI_TYPE_SINGLE_COMPLEX) *dt = MPI_C_FLOAT_COMPLEX; + else if(pdt == PAMI_TYPE_DOUBLE_COMPLEX) *dt = MPI_C_DOUBLE_COMPLEX; + else if(isPAMI_LOC_TYPE(pdt)) + { + if(pdt == PAMI_TYPE_LOC_2INT) *dt = MPI_2INT; + else if(pdt == PAMI_TYPE_LOC_FLOAT_INT) *dt = MPI_FLOAT_INT; + else if(pdt == PAMI_TYPE_LOC_DOUBLE_INT) *dt = MPI_DOUBLE_INT; + else if(pdt == PAMI_TYPE_LOC_SHORT_INT) *dt = MPI_SHORT_INT; + else if(pdt == PAMI_TYPE_LOC_LONG_INT) *dt = MPI_LONG_INT; + else if(pdt == PAMI_TYPE_LOC_LONGDOUBLE_INT)*dt = MPI_LONG_DOUBLE_INT; + else if(pdt == PAMI_TYPE_LOC_2FLOAT) *dt = MPI_2REAL; + else if(pdt == PAMI_TYPE_LOC_2DOUBLE) *dt = MPI_2DOUBLE_PRECISION; + + if(pop == NULL) return MPI_SUCCESS; + if(pop == PAMI_DATA_MINLOC) + { + *op = MPI_MINLOC; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_MAXLOC) + { + *op = MPI_MAXLOC; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_COPY) + { + *op = MPI_REPLACE; + return MPI_SUCCESS; + } + else return -1; + } + else if(pdt == PAMI_TYPE_LOGICAL4) + { + *dt = MPI_LOGICAL; + if(pop == NULL) return MPI_SUCCESS; + if(pop == PAMI_DATA_LOR) + { + *op = MPI_LOR; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_LAND) + { + *op = MPI_LAND; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_LXOR) + { + *op = MPI_LXOR; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_COPY) + { + *op = MPI_REPLACE; + return MPI_SUCCESS; + } + return -1; + } + else if(pdt == PAMI_TYPE_LOGICAL1) + { + *dt = MPI_C_BOOL; + if(pop == NULL) return MPI_SUCCESS; + if(pop == PAMI_DATA_LOR) + { + *op = MPI_LOR; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_LAND) + { + *op = MPI_LAND; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_LXOR) + { + *op = MPI_LXOR; + return MPI_SUCCESS; + } + if(pop == PAMI_DATA_COPY) + { + *op = MPI_REPLACE; + return MPI_SUCCESS; + } + return -1; + } + + + if(*dt == MPI_DATATYPE_NULL) return -1; + + if(pop == NULL) return MPI_SUCCESS; /* just doing DT conversion */ + + *op = MPI_OP_NULL; + if(pop == PAMI_DATA_SUM) *op = MPI_SUM; + else if(pop ==PAMI_DATA_PROD) *op = MPI_PROD; + else if(pop ==PAMI_DATA_MAX) *op = MPI_MAX; + else if(pop ==PAMI_DATA_MIN) *op = MPI_MIN; + else if(pop ==PAMI_DATA_BAND) *op = MPI_BAND; + else if(pop ==PAMI_DATA_BOR) *op = MPI_BOR; + else if(pop ==PAMI_DATA_BXOR) *op = MPI_BXOR; + else if(pop ==PAMI_DATA_LAND) *op = MPI_LAND; + else if(pop ==PAMI_DATA_LOR) *op = MPI_LOR; + else if(pop ==PAMI_DATA_LXOR) *op = MPI_LXOR; + else if(pop ==PAMI_DATA_COPY) *op = MPI_REPLACE; + + if(*op == MPI_OP_NULL) return -1; + + return MPI_SUCCESS; +} + diff --git a/src/mpid/pamid/src/coll/gather/mpido_gather.c b/src/mpid/pamid/src/coll/gather/mpido_gather.c index acdca25..b0e1ca2 100644 --- a/src/mpid/pamid/src/coll/gather/mpido_gather.c +++ b/src/mpid/pamid/src/coll/gather/mpido_gather.c @@ -446,3 +446,36 @@ int MPIDO_Gather_simple(const void *sendbuf, TRACE_ERR("Leaving MPIDO_Gather_optimized\n"); return MPI_SUCCESS; } + +int +MPIDO_CSWrapper_gather(pami_xfer_t *gather, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *sbuf; + MPIDI_coll_check_in_place(gather->cmd.xfer_gather.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( gather->cmd.xfer_gather.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( gather->cmd.xfer_gather.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Gather(sbuf, + gather->cmd.xfer_gather.stypecount, sendtype, + gather->cmd.xfer_gather.rcvbuf, + gather->cmd.xfer_gather.rtypecount, recvtype, + gather->cmd.xfer_gather.root, comm_ptr, &mpierrno); + if(gather->cb_done && rc == 0) + gather->cb_done(NULL, gather->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c b/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c index 1645f44..d04b567 100644 --- a/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c +++ b/src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c @@ -298,10 +298,6 @@ int MPIDO_Gatherv_simple(const void *sendbuf, root, comm_ptr, mpierrno); } - - - - gatherv.cb_done = cb_gatherv; gatherv.cookie = (void *)&gatherv_active; gatherv.cmd.xfer_gatherv_int.root = MPID_VCR_GET_LPID(comm_ptr->vcr, root); @@ -331,3 +327,37 @@ int MPIDO_Gatherv_simple(const void *sendbuf, TRACE_ERR("Leaving MPIDO_Gatherv_optimized\n"); return MPI_SUCCESS; } + +int +MPIDO_CSWrapper_gatherv(pami_xfer_t *gatherv, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *sbuf; + MPIDI_coll_check_in_place(gatherv->cmd.xfer_gatherv_int.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( gatherv->cmd.xfer_gatherv_int.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( gatherv->cmd.xfer_gatherv_int.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Gatherv(sbuf, + gatherv->cmd.xfer_gatherv_int.stypecount, sendtype, + gatherv->cmd.xfer_gatherv_int.rcvbuf, + gatherv->cmd.xfer_gatherv_int.rtypecounts, + gatherv->cmd.xfer_gatherv_int.rdispls, recvtype, + gatherv->cmd.xfer_gatherv_int.root, comm_ptr, &mpierrno); + if(gatherv->cb_done && rc == 0) + gatherv->cb_done(NULL, gatherv->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/reduce/mpido_reduce.c b/src/mpid/pamid/src/coll/reduce/mpido_reduce.c index 45fb05c..143ec74 100644 --- a/src/mpid/pamid/src/coll/reduce/mpido_reduce.c +++ b/src/mpid/pamid/src/coll/reduce/mpido_reduce.c @@ -305,3 +305,32 @@ int MPIDO_Reduce_simple(const void *sendbuf, TRACE_ERR("Reduce done\n"); return MPI_SUCCESS; } + + +int +MPIDO_CSWrapper_reduce(pami_xfer_t *reduce, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype type; + MPI_Op op; + void *sbuf; + MPIDI_coll_check_in_place(reduce->cmd.xfer_reduce.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( reduce->cmd.xfer_reduce.stype, + &type, + reduce->cmd.xfer_reduce.op, + &op); + if(rc == -1) return rc; + + + rc = MPIR_Reduce(sbuf, + reduce->cmd.xfer_reduce.rcvbuf, + reduce->cmd.xfer_reduce.rtypecount, type, op, + reduce->cmd.xfer_reduce.root, comm_ptr, &mpierrno); + if(reduce->cb_done && rc == 0) + reduce->cb_done(NULL, reduce->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/scan/mpido_scan.c b/src/mpid/pamid/src/coll/scan/mpido_scan.c index f47cb81..b9e848e 100644 --- a/src/mpid/pamid/src/coll/scan/mpido_scan.c +++ b/src/mpid/pamid/src/coll/scan/mpido_scan.c @@ -292,3 +292,36 @@ int MPIDO_Scan_simple(const void *sendbuf, void *recvbuf, return MPIDO_Doscan_simple(sendbuf, recvbuf, count, datatype, op, comm_ptr, mpierrno, 0); } + +int +MPIDO_CSWrapper_scan(pami_xfer_t *scan, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype type; + MPI_Op op; + void *sbuf; + MPIDI_coll_check_in_place(scan->cmd.xfer_scan.sndbuf, &sbuf); + int rc = MPIDI_Dtpami_to_dtmpi( scan->cmd.xfer_scan.stype, + &type, + scan->cmd.xfer_scan.op, + &op); + if(rc == -1) return rc; + + if(scan->cmd.xfer_scan.exclusive) + rc = MPIR_Exscan(sbuf, + scan->cmd.xfer_scan.rcvbuf, + scan->cmd.xfer_scan.rtypecount, type, op, + comm_ptr, &mpierrno); + else + rc = MPIR_Scan(sbuf, + scan->cmd.xfer_scan.rcvbuf, + scan->cmd.xfer_scan.rtypecount, type, op, + comm_ptr, &mpierrno); + if(scan->cb_done && rc == 0) + scan->cb_done(NULL, scan->cookie, PAMI_SUCCESS); + return rc; + +} + diff --git a/src/mpid/pamid/src/coll/scatter/mpido_scatter.c b/src/mpid/pamid/src/coll/scatter/mpido_scatter.c index 082c316..e5aad3b 100644 --- a/src/mpid/pamid/src/coll/scatter/mpido_scatter.c +++ b/src/mpid/pamid/src/coll/scatter/mpido_scatter.c @@ -420,6 +420,38 @@ int MPIDO_Scatter_simple(const void *sendbuf, } +int +MPIDO_CSWrapper_scatter(pami_xfer_t *scatter, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *rbuf; + MPIDI_coll_check_in_place(scatter->cmd.xfer_scatter.rcvbuf, &rbuf); + int rc = MPIDI_Dtpami_to_dtmpi( scatter->cmd.xfer_scatter.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( scatter->cmd.xfer_scatter.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Scatter(scatter->cmd.xfer_scatter.sndbuf, + scatter->cmd.xfer_scatter.stypecount, sendtype, + rbuf, + scatter->cmd.xfer_scatter.rtypecount, recvtype, + scatter->cmd.xfer_scatter.root, comm_ptr, &mpierrno); + if(scatter->cb_done && rc == 0) + scatter->cb_done(NULL, scatter->cookie, PAMI_SUCCESS); + return rc; + +} + #if 0 /* old glue-based scatter-via-bcast */ /* Assume glue protocol sucks for now.... Needs work if not or to enable */ diff --git a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c index 508739a..51b01ec 100644 --- a/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c +++ b/src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c @@ -499,7 +499,7 @@ int MPIDO_Scatterv_simple(const void *sendbuf, scatterv.cmd.xfer_scatterv_int.root = MPID_VCR_GET_LPID(comm_ptr->vcr, root); scatterv.algorithm = mpid->coll_algorithm[PAMI_XFER_SCATTERV_INT][0][0]; - my_scatterv_md = &mpid->coll_metadata[PAMI_XFER_SCATTERV][0][0]; + my_scatterv_md = &mpid->coll_metadata[PAMI_XFER_SCATTERV_INT][0][0]; scatterv.cmd.xfer_scatterv_int.rcvbuf = rbuf; scatterv.cmd.xfer_scatterv_int.sndbuf = sbuf; @@ -526,6 +526,40 @@ int MPIDO_Scatterv_simple(const void *sendbuf, } +int +MPIDO_CSWrapper_scatterv(pami_xfer_t *scatterv, + void *comm) +{ + int mpierrno = 0; + MPID_Comm *comm_ptr = (MPID_Comm*)comm; + MPI_Datatype sendtype, recvtype; + void *rbuf; + MPIDI_coll_check_in_place(scatterv->cmd.xfer_scatterv_int.rcvbuf, &rbuf); + int rc = MPIDI_Dtpami_to_dtmpi( scatterv->cmd.xfer_scatterv_int.stype, + &sendtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIDI_Dtpami_to_dtmpi( scatterv->cmd.xfer_scatterv_int.rtype, + &recvtype, + NULL, + NULL); + if(rc == -1) return rc; + + rc = MPIR_Scatterv(scatterv->cmd.xfer_scatterv_int.sndbuf, + scatterv->cmd.xfer_scatterv_int.stypecounts, + scatterv->cmd.xfer_scatterv_int.sdispls, sendtype, + rbuf, + scatterv->cmd.xfer_scatterv_int.rtypecount, recvtype, + scatterv->cmd.xfer_scatterv_int.root, comm_ptr, &mpierrno); + if(scatterv->cb_done && rc == 0) + scatterv->cb_done(NULL, scatterv->cookie, PAMI_SUCCESS); + return rc; + +} + + #if 0 /* remove the glue-based optimized scattervs for now. */ diff --git a/src/mpid/pamid/src/comm/mpid_comm.c b/src/mpid/pamid/src/comm/mpid_comm.c index 989dc19..844f0cd 100644 --- a/src/mpid/pamid/src/comm/mpid_comm.c +++ b/src/mpid/pamid/src/comm/mpid_comm.c @@ -50,6 +50,93 @@ int MPIDI_Comm_destroy (MPID_Comm *comm) return MPI_SUCCESS; } +pami_result_t MPIDI_Comm_create_from_pami_geom(pami_geometry_range_t *task_slices, + size_t slice_count, + pami_geometry_t *geometry, + void **cookie) +{ + int mpi_errno = MPI_SUCCESS; + int num_tasks = 0; + int *ranks = NULL; + MPID_Comm *comm_ptr = NULL, *new_comm_ptr = NULL; + MPID_Group *group_ptr = NULL, *new_group_ptr = NULL; + int i = 0, j = 0; + MPIR_Context_id_t new_context_id = 0; + int *mapping = NULL; + + /* Get comm_ptr for MPI_COMM_WORLD and get the group_ptr for it */ + MPID_Comm_get_ptr( MPI_COMM_WORLD, comm_ptr ); + mpi_errno = MPIR_Comm_group_impl(comm_ptr, &group_ptr); + if (mpi_errno) + { + TRACE_ERR("Error while creating group_ptr from MPI_COMM_WORLD in MPIDI_Comm_create_from_pami_geom\n"); + return PAMI_ERROR; + } + + /* Create the ranks list from the pami_geometry_range_t array */ + for(i = 0; i < slice_count; i++) + { + num_tasks += (task_slices[i].hi - task_slices[i].lo) + 1; + } + ranks = MPIU_Calloc0(num_tasks, int); + for(i = 0; i < slice_count; i++) + { + int slice_sz = (task_slices[i].hi - task_slices[i].lo) + 1; + int k = 0; + for(k = 0; k < slice_sz; k++) + { + ranks[j] = task_slices[i].lo + k; + j++; + } + } + + /* Now we have all we need to create the new group. Create it */ + mpi_errno = MPIR_Group_incl_impl(group_ptr, num_tasks, ranks, &new_group_ptr); + if (mpi_errno) + { + TRACE_ERR("Error while creating new_group_ptr from group_ptr in MPIDI_Comm_create_from_pami_geom\n"); + return PAMI_ERROR; + } + + /* Now create the communicator using the new_group_ptr */ + mpi_errno = MPIR_Comm_create_intra(comm_ptr, new_group_ptr, &new_comm_ptr); + if (mpi_errno) + { + TRACE_ERR("Error while creating new_comm_ptr from group_ptr in MPIDI_Comm_create_from_pami_geom\n"); + return PAMI_ERROR; + } + + if(new_comm_ptr) + { + /* Get the geometry from the communicator and set the out parameters */ + *geometry = new_comm_ptr->mpid.geometry; + *cookie = new_comm_ptr; + } + else + { + *geometry = PAMI_GEOMETRY_NULL; + *cookie = NULL; + } + + /* Cleanup */ + MPIU_TestFree(&ranks); + + return PAMI_SUCCESS; +} + +pami_result_t MPIDI_Comm_destroy_external(void *comm_ext) +{ + int mpi_errno = 0; + MPID_Comm* comm_ptr = (MPID_Comm*)comm_ext; + mpi_errno = MPIR_Comm_free_impl(comm_ptr); + if (mpi_errno) + { + TRACE_ERR("Error while destroying comm_ptr in MPIDI_Comm_destroy_external\n"); + return PAMI_ERROR; + } + return PAMI_SUCCESS; +} + typedef struct MPIDI_Post_geom_create { pami_work_t state; @@ -289,6 +376,15 @@ void MPIDI_Coll_comm_destroy(MPID_Comm *comm) for(i=0;i<PAMI_XFER_COUNT;i++) { TRACE_ERR("Freeing algo/meta %d\n", i); + /* When allocating comm->mpid.coll_algorithm, we skip allocations for + AM collectives. Also there is no explicit initialization of + comm->mpid.coll_algorithm to NULLs. This may cause MPIU_TestFree to + cause problems when freeing. We skip AM collectives here as we skip + allocating them in MPIDI_Comm_coll_query */ + if(i == PAMI_XFER_AMBROADCAST || i == PAMI_XFER_AMSCATTER || + i == PAMI_XFER_AMGATHER || i == PAMI_XFER_AMREDUCE) + continue; + MPIU_TestFree(&comm->mpid.coll_algorithm[i][0]); MPIU_TestFree(&comm->mpid.coll_algorithm[i][1]); MPIU_TestFree(&comm->mpid.coll_metadata[i][0]); diff --git a/src/mpid/pamid/src/comm/mpid_selectcolls.c b/src/mpid/pamid/src/comm/mpid_selectcolls.c index d4c50df..834542f 100644 --- a/src/mpid/pamid/src/comm/mpid_selectcolls.c +++ b/src/mpid/pamid/src/comm/mpid_selectcolls.c @@ -24,6 +24,68 @@ #include <mpidimpl.h> +pami_metadata_t ext_metadata; +advisor_algorithm_t ext_algorithms[1]; +external_algorithm_t ext_algorithm; + +#define MPIDI_UPDATE_COLLSEL_EXT_ALGO(cb,nm,xfer_type) { \ + ext_algorithm.callback = cb; \ + ext_algorithm.cookie = comm; \ + ext_metadata.name = nm; \ + ext_algorithms[0].algorithm.external = ext_algorithm; \ + ext_algorithms[0].metadata = &ext_metadata; \ + ext_algorithms[0].algorithm_type = COLLSEL_EXTERNAL_ALGO; \ + collsel_register_algorithms(MPIDI_Collsel_advisor_table, \ + comm->mpid.geometry, \ + xfer_type, \ + &ext_algorithms[0], \ + 1); \ +} + + +pami_result_t MPIDI_Register_algorithms_ext(void *cookie, + pami_xfer_type_t collective, + advisor_algorithm_t **algorithms, + size_t *num_algorithms) +{ + external_algorithm_fn callback; + char *algoname; + + switch(collective) + { + case PAMI_XFER_BROADCAST: callback = MPIDO_CSWrapper_bcast; algoname = "EXT:Bcast:P2P:P2P"; break; + case PAMI_XFER_ALLREDUCE: callback = MPIDO_CSWrapper_allreduce; algoname = "EXT:Allreduce:P2P:P2P"; break; + case PAMI_XFER_REDUCE: callback = MPIDO_CSWrapper_reduce; algoname = "EXT:Reduce:P2P:P2P"; break; + case PAMI_XFER_ALLGATHER: callback = MPIDO_CSWrapper_allgather; algoname = "EXT:Allgather:P2P:P2P"; break; + case PAMI_XFER_ALLGATHERV_INT: callback = MPIDO_CSWrapper_allgatherv; algoname = "EXT:Allgatherv:P2P:P2P"; break; + case PAMI_XFER_SCATTER: callback = MPIDO_CSWrapper_scatter; algoname = "EXT:Scatter:P2P:P2P"; break; + case PAMI_XFER_SCATTERV_INT: callback = MPIDO_CSWrapper_scatterv; algoname = "EXT:Scatterv:P2P:P2P"; break; + case PAMI_XFER_GATHER: callback = MPIDO_CSWrapper_gather; algoname = "EXT:Gather:P2P:P2P"; break; + case PAMI_XFER_GATHERV_INT: callback = MPIDO_CSWrapper_gatherv; algoname = "EXT:Gatherv:P2P:P2P"; break; + case PAMI_XFER_BARRIER: callback = MPIDO_CSWrapper_barrier; algoname = "EXT:Barrier:P2P:P2P"; break; + case PAMI_XFER_ALLTOALL: callback = MPIDO_CSWrapper_alltoall; algoname = "EXT:Alltoall:P2P:P2P"; break; + case PAMI_XFER_ALLTOALLV_INT: callback = MPIDO_CSWrapper_alltoallv; algoname = "EXT:Alltoallv:P2P:P2P"; break; + case PAMI_XFER_SCAN: callback = MPIDO_CSWrapper_scan; algoname = "EXT:Scan:P2P:P2P"; break; + case PAMI_XFER_ALLGATHERV: + case PAMI_XFER_SCATTERV: + case PAMI_XFER_GATHERV: + case PAMI_XFER_ALLTOALLV: + case PAMI_XFER_REDUCE_SCATTER: + *num_algorithms = 0; + return PAMI_SUCCESS; + default: return -1; + } + *num_algorithms = 1; + ext_algorithm.callback = callback; + ext_algorithm.cookie = cookie; + ext_metadata.name = algoname; + ext_algorithms[0].algorithm.external = ext_algorithm; + ext_algorithms[0].metadata = &ext_metadata; + ext_algorithms[0].algorithm_type = COLLSEL_EXTERNAL_ALGO; + *algorithms = &ext_algorithms[0]; + return PAMI_SUCCESS; +} + static char* MPIDI_Coll_type_name(int i) { switch(i) @@ -417,89 +479,106 @@ void MPIDI_Comm_coll_envvars(MPID_Comm *comm) /* If automatic collective selection is enabled and user didn't specifically overwrite it, then use auto coll sel.. Otherwise, go through the manual coll sel code path. */ - /* ************ Barrier ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_BARRIER) && - comm->mpid.user_selected_type[PAMI_XFER_BARRIER] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Barrier = MPIDO_Barrier_simple; - } - /* ************ Bcast ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_BCAST) && - comm->mpid.user_selected_type[PAMI_XFER_BROADCAST] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Bcast = MPIDO_Bcast_simple; - } - /* ************ Allreduce ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLREDUCE) && - comm->mpid.user_selected_type[PAMI_XFER_ALLREDUCE] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Allreduce = MPIDO_Allreduce_simple; - } - /* ************ Allgather ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLGATHER) && - comm->mpid.user_selected_type[PAMI_XFER_ALLGATHER] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Allgather = MPIDO_Allgather_simple; - } - /* ************ Allgatherv ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLGATHERV) && - comm->mpid.user_selected_type[PAMI_XFER_ALLGATHERV_INT] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Allgatherv = MPIDO_Allgatherv_simple; - } - /* ************ Scatterv ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_SCATTERV) && - comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Scatterv = MPIDO_Scatterv_simple; - } - /* ************ Scatter ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_SCATTER) && - comm->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Scatter = MPIDO_Scatter_simple; - } - /* ************ Gather ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_GATHER) && - comm->mpid.user_selected_type[PAMI_XFER_GATHER] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Gather = MPIDO_Gather_simple; - } - /* ************ Alltoallv ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLTOALLV) && - comm->mpid.user_selected_type[PAMI_XFER_ALLTOALLV_INT] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Alltoallv = MPIDO_Alltoallv_simple; - } - /* ************ Alltoall ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLTOALL) && - comm->mpid.user_selected_type[PAMI_XFER_ALLTOALL] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Alltoall = MPIDO_Alltoall_simple; - } - /* ************ Gatherv ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_GATHERV) && - comm->mpid.user_selected_type[PAMI_XFER_GATHERV_INT] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Gatherv = MPIDO_Gatherv_simple; - } - /* ************ Reduce ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_REDUCE) && - comm->mpid.user_selected_type[PAMI_XFER_REDUCE] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Reduce = MPIDO_Reduce_simple; - } - /* ************ Scan ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_SCAN) && - comm->mpid.user_selected_type[PAMI_XFER_SCAN] == MPID_COLL_NOSELECTION) + if(MPIDI_Process.optimized.auto_select_colls != MPID_AUTO_SELECT_COLLS_NONE) { - comm->coll_fns->Scan = MPIDO_Scan_simple; - } - /* ************ Exscan ************ */ - if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_EXSCAN) && - comm->mpid.user_selected_type[PAMI_XFER_SCAN] == MPID_COLL_NOSELECTION) - { - comm->coll_fns->Exscan = MPIDO_Exscan_simple; + pami_extension_collsel_register_algorithms collsel_register_algorithms = + (pami_extension_collsel_register_algorithms) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_register_algorithms"); + if(collsel_register_algorithms != NULL) + { + + /* ************ Barrier ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_BARRIER) && + comm->mpid.user_selected_type[PAMI_XFER_BARRIER] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Barrier = MPIDO_Barrier_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_barrier,"EXT:Barrier:P2P:P2P",PAMI_XFER_BARRIER); + } + /* ************ Bcast ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_BCAST) && + comm->mpid.user_selected_type[PAMI_XFER_BROADCAST] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Bcast = MPIDO_Bcast_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_bcast,"EXT:Bcast:P2P:P2P",PAMI_XFER_BROADCAST); + } + /* ************ Allreduce ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLREDUCE) && + comm->mpid.user_selected_type[PAMI_XFER_ALLREDUCE] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Allreduce = MPIDO_Allreduce_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_allreduce,"EXT:Allreduce:P2P:P2P",PAMI_XFER_ALLREDUCE); + } + /* ************ Allgather ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLGATHER) && + comm->mpid.user_selected_type[PAMI_XFER_ALLGATHER] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Allgather = MPIDO_Allgather_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_allgather,"EXT:Allgather:P2P:P2P",PAMI_XFER_ALLGATHER); + } + /* ************ Allgatherv ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLGATHERV) && + comm->mpid.user_selected_type[PAMI_XFER_ALLGATHERV_INT] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Allgatherv = MPIDO_Allgatherv_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_allgatherv,"EXT:Allgatherv:P2P:P2P",PAMI_XFER_ALLGATHERV_INT); + } + /* ************ Scatterv ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_SCATTERV) && + comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Scatterv = MPIDO_Scatterv_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_scatterv,"EXT:Scatterv:P2P:P2P",PAMI_XFER_SCATTERV_INT); + } + /* ************ Scatter ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_SCATTER) && + comm->mpid.user_selected_type[PAMI_XFER_SCATTER] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Scatter = MPIDO_Scatter_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_scatter,"EXT:Scatter:P2P:P2P",PAMI_XFER_SCATTER); + } + /* ************ Gather ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_GATHER) && + comm->mpid.user_selected_type[PAMI_XFER_GATHER] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Gather = MPIDO_Gather_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_gather,"EXT:Gather:P2P:P2P",PAMI_XFER_GATHER); + } + /* ************ Alltoallv ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLTOALLV) && + comm->mpid.user_selected_type[PAMI_XFER_ALLTOALLV_INT] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Alltoallv = MPIDO_Alltoallv_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_alltoallv,"EXT:Alltoallv:P2P:P2P",PAMI_XFER_ALLTOALLV_INT); + } + /* ************ Alltoall ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_ALLTOALL) && + comm->mpid.user_selected_type[PAMI_XFER_ALLTOALL] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Alltoall = MPIDO_Alltoall_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_alltoall,"EXT:Alltoall:P2P:P2P",PAMI_XFER_ALLTOALL); + } + /* ************ Gatherv ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_GATHERV) && + comm->mpid.user_selected_type[PAMI_XFER_GATHERV_INT] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Gatherv = MPIDO_Gatherv_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_gatherv,"EXT:Gatherv:P2P:P2P",PAMI_XFER_GATHERV_INT); + } + /* ************ Reduce ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_REDUCE) && + comm->mpid.user_selected_type[PAMI_XFER_REDUCE] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Reduce = MPIDO_Reduce_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_reduce,"EXT:Reduce:P2P:P2P",PAMI_XFER_REDUCE); + } + /* ************ Scan ************ */ + if((MPIDI_Process.optimized.auto_select_colls & MPID_AUTO_SELECT_COLLS_SCAN) && + comm->mpid.user_selected_type[PAMI_XFER_SCAN] == MPID_COLL_NOSELECTION) + { + comm->coll_fns->Scan = MPIDO_Scan_simple; + MPIDI_UPDATE_COLLSEL_EXT_ALGO(MPIDO_CSWrapper_scan,"EXT:Scan:P2P:P2P",PAMI_XFER_SCAN); + } + } } TRACE_ERR("MPIDI_Comm_coll_envvars exit\n"); } diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 39d54af..5fbba71 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -45,8 +45,8 @@ int mpidi_dynamic_tasking = 0; pami_extension_t pe_extension; #endif -pami_client_t MPIDI_Client; -pami_context_t MPIDI_Context[MPIDI_MAX_CONTEXTS]; +pami_client_t MPIDI_Client; +pami_context_t MPIDI_Context[MPIDI_MAX_CONTEXTS]; MPIDI_Process_t MPIDI_Process = { .verbose = 0, @@ -274,6 +274,14 @@ static struct #endif }; +/* ------------------------------ */ +/* Collective selection extension */ +/* ------------------------------ */ +pami_extension_t MPIDI_Collsel_extension; +advisor_t MPIDI_Collsel_advisor; +advisor_table_t MPIDI_Collsel_advisor_table; +advisor_params_t MPIDI_Collsel_advisor_params; +char *MPIDI_Collsel_output_file; static void MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char **world_tasks) { @@ -385,6 +393,145 @@ MPIDI_PAMI_client_init(int* rank, int* size, int* mpidi_dynamic_tasking, char ** } } +void MPIDI_Init_collsel_extension() +{ + pami_result_t status = PAMI_ERROR; + status = PAMI_Extension_open (MPIDI_Client, "EXT_collsel", &MPIDI_Collsel_extension); + if(status == PAMI_SUCCESS) + { + if(MPIDI_Process.optimized.auto_select_colls == MPID_AUTO_SELECT_COLLS_TUNE) + { + advisor_configuration_t configuration[1]; + pami_extension_collsel_init pamix_collsel_init = + (pami_extension_collsel_init) PAMI_Extension_symbol (MPIDI_Collsel_extension, "Collsel_init"); + status = pamix_collsel_init (MPIDI_Client, configuration, 1, &MPIDI_Context[0], 1, &MPIDI_Collsel_advisor); + if(status != PAMI_SUCCESS) + { + fprintf (stderr, "Error. The collsel_init failed. result = %d\n", status); + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; + } + + } + else if(MPIDI_Process.optimized.auto_select_colls == MPID_AUTO_SELECT_COLLS_ALL) + { + pami_extension_collsel_initialized pamix_collsel_initialized = + (pami_extension_collsel_initialized) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_initialized"); + if(pamix_collsel_initialized(MPIDI_Client, &MPIDI_Collsel_advisor) == 1) + { + char *collselfile; + collselfile = getenv("MP_COLLECTIVE_SELECTION_FILE"); + pami_extension_collsel_table_load pamix_collsel_table_load = + (pami_extension_collsel_table_load) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_table_load"); + if(collselfile != NULL) + status = pamix_collsel_table_load(MPIDI_Collsel_advisor, collselfile, &MPIDI_Collsel_advisor_table); + else + status = pamix_collsel_table_load(MPIDI_Collsel_advisor, "pami_tune_results.xml", &MPIDI_Collsel_advisor_table); + if (status == PAMI_SUCCESS) + { + pami_xfer_type_t *collsel_collectives = NULL; + unsigned num_collectives; + pami_extension_collsel_get_collectives pamix_collsel_get_collectives = + (pami_extension_collsel_get_collectives) PAMI_Extension_symbol(MPIDI_Collsel_extension, + "Collsel_get_collectives"); + status = pamix_collsel_get_collectives(MPIDI_Collsel_advisor_table, &collsel_collectives, &num_collectives); + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; + if(collsel_collectives != NULL) + { + unsigned i = 0; + for(i = 0; i < num_collectives; i++) + { + switch(collsel_collectives[i]) + { + case PAMI_XFER_BROADCAST: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_BCAST; + break; + case PAMI_XFER_ALLREDUCE: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_ALLREDUCE; + break; + case PAMI_XFER_REDUCE: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_REDUCE; + break; + case PAMI_XFER_ALLGATHER: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_ALLGATHER; + break; + case PAMI_XFER_ALLGATHERV: + case PAMI_XFER_ALLGATHERV_INT: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_ALLGATHERV; + break; + case PAMI_XFER_SCATTER: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_SCATTER; + break; + case PAMI_XFER_SCATTERV: + case PAMI_XFER_SCATTERV_INT: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_SCATTERV; + break; + case PAMI_XFER_GATHER: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_GATHER; + break; + case PAMI_XFER_GATHERV: + case PAMI_XFER_GATHERV_INT: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_GATHERV; + break; + case PAMI_XFER_BARRIER: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_BARRIER; + break; + case PAMI_XFER_ALLTOALL: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_ALLTOALL; + break; + case PAMI_XFER_ALLTOALLV: + case PAMI_XFER_ALLTOALLV_INT: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_ALLTOALLV; + break; + case PAMI_XFER_SCAN: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_SCAN; + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_EXSCAN; + break; + case PAMI_XFER_REDUCE_SCATTER: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_REDUCE_SCATTER; + break; + default: + MPIDI_Process.optimized.auto_select_colls |= MPID_AUTO_SELECT_COLLS_NONE; + } + } + MPIU_Free(collsel_collectives); + } + } + else + { + fprintf (stderr, "Error. Collsel_table_load failed. result = %d\n", status); + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; + } + } + else + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; + } + else + PAMI_Extension_close(MPIDI_Collsel_extension); + } + else + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; +} + +void MPIDI_Collsel_table_generate() +{ + external_geometry_ops_t external_ops; + external_ops.geometry_create = MPIDI_Comm_create_from_pami_geom; + external_ops.geometry_destroy = MPIDI_Comm_destroy_external; + external_ops.register_algorithms = MPIDI_Register_algorithms_ext; + pami_result_t status = PAMI_SUCCESS; + pami_extension_collsel_table_generate pamix_collsel_table_generate = + (pami_extension_collsel_table_generate) PAMI_Extension_symbol (MPIDI_Collsel_extension, "Collsel_table_generate"); + + status = pamix_collsel_table_generate (MPIDI_Collsel_advisor, MPIDI_Collsel_output_file, &MPIDI_Collsel_advisor_params, &external_ops, 1); + if(status != PAMI_SUCCESS) + { + fprintf (stderr, "Error. The collsel_table_generate failed. result = %d\n", status); + } + +} + static void MPIDI_PAMI_context_init(int* threading, int *size) @@ -494,7 +641,7 @@ MPIDI_PAMI_context_init(int* threading, int *size) #ifdef MPIDI_TRACE - int i; + int i; MPIDI_Trace_buf = MPIU_Calloc0(numTasks, MPIDI_Trace_buf_t); if(MPIDI_Trace_buf == NULL) MPID_abort(); memset((void *) MPIDI_Trace_buf,0, sizeof(MPIDI_Trace_buf_t)); @@ -529,6 +676,14 @@ MPIDI_PAMI_context_init(int* threading, int *size) rc = PAMI_Context_createv(MPIDI_Client, config, cfgval, MPIDI_Context, MPIDI_Process.avail_contexts); MPID_assert_always(rc == PAMI_SUCCESS); + + /* --------------------------------------------- */ + /* Get collective selection advisor and cache it */ + /* --------------------------------------------- */ + /* Context is created, i.e. collective selection extension is initialized in PAMI. Now I can get the + advisor if I am not in TUNE mode. If in TUNE mode, I can init collsel and generate the table */ + MPIDI_Init_collsel_extension(); + #if (MPIDI_STATISTICS || MPIDI_PRINTENV) MPIDI_open_pe_extension(); #endif @@ -731,7 +886,7 @@ MPIDI_PAMI_init(int* rank, int* size, int* threading) MPIDI_Process.rma_pending, MPIDI_Process.shmem_pt2pt, MPIDI_Process.disable_internal_eager_scale, -#if TOKEN_FLOW_CONTROL +#if TOKEN_FLOW_CONTROL MPIDI_Process.mp_buf_mem, MPIDI_Process.mp_buf_mem_max, MPIDI_Process.is_token_flow_control_on, @@ -1089,6 +1244,26 @@ int MPID_Init(int * argc, MPIDI_Print_mpenv(rank,size); } #endif + /* ----------------------------------------------- */ + /* parse params for pami_tune if in benchmark mode */ + /* ----------------------------------------------- */ + if(MPIDI_Process.optimized.auto_select_colls == MPID_AUTO_SELECT_COLLS_TUNE) + { + if(argc != NULL && argv != NULL) + { + if(MPIDI_collsel_pami_tune_parse_params(*argc, *argv) != PAMI_SUCCESS) + { + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; + } + } + else + { + if(MPIDI_collsel_pami_tune_parse_params(0, NULL) != PAMI_SUCCESS) + { + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; + } + } + } return MPI_SUCCESS; } @@ -1100,6 +1275,14 @@ int MPID_InitCompleted() { MPIDI_NBC_init(); MPIDI_Progress_init(); + /* ----------------------------------------------- */ + /* Now all is ready.. call table generate */ + /* ----------------------------------------------- */ + if(MPIDI_Process.optimized.auto_select_colls == MPID_AUTO_SELECT_COLLS_TUNE) + { + MPIDI_Collsel_table_generate(); + MPIDI_collsel_pami_tune_cleanup(); + } return MPI_SUCCESS; } @@ -1182,7 +1365,7 @@ int MPIDI_Banner(char * bufPtr) { MPIU_Free(tmx); return MPI_SUCCESS; } -#endif +#endif static inline void diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c index 3e25f06..f08acb7 100644 --- a/src/mpid/pamid/src/mpidi_env.c +++ b/src/mpid/pamid/src/mpidi_env.c @@ -456,7 +456,9 @@ ENV_Char__(char* name[], unsigned* val, char* string) break; } - if ((env[0]=='y')|| (env[0]=='Y')|| (env[0]=='p')|| (env[0]=='P') || (env[0]=='F')|| (env[0]=='f')) + if ( (env[0]=='y')|| (env[0]=='Y') + || (env[0]=='p')|| (env[0]=='P') + || (env[0]=='F')|| (env[0]=='f')) *val = 1; /*This may seem redundant; however, in some cases we need to force val=0 if value = no/none*/ @@ -864,22 +866,35 @@ MPIDI_Env_setup(int rank, int requested) /* Finally, if MP_COLLECTIVE_SELECTION is "on", then we want to overwrite any other setting */ { - unsigned temp; - temp = 0; - char* names[] = {"MP_COLLECTIVE_SELECTION", NULL}; - ENV_Char(names, &temp); - if(temp) + char *env = getenv("MP_COLLECTIVE_SELECTION"); + if(env != NULL) { pami_extension_t extension; pami_result_t status = PAMI_ERROR; status = PAMI_Extension_open (MPIDI_Client, "EXT_collsel", &extension); if(status == PAMI_SUCCESS) { + char *env = getenv("MP_COLLECTIVE_SELECTION"); + if(env != NULL) + { + if(strncasecmp(env, "TUN", 3) == 0) + { + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_TUNE; + MPIDI_Process.optimized.collectives = 1; + } + else if(strncasecmp(env, "YES", 3) == 0) + { + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_ALL; /* All collectives will be using auto coll sel. + We will check later on each individual coll. */ + MPIDI_Process.optimized.collectives = 1; + } + else + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE; - MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_ALL; /* All collectives will be using auto coll sel. - We will check later on each individual coll. */ - MPIDI_Process.optimized.collectives = 1; /* Enable optimized collectives so we can create PAMI Geometry */ + } } + else + MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE;/* Auto coll sel is disabled for all */ } else MPIDI_Process.optimized.auto_select_colls = MPID_AUTO_SELECT_COLLS_NONE;/* Auto coll sel is disabled for all */ diff --git a/src/mpid/pamid/src/mpidi_util.c b/src/mpid/pamid/src/mpidi_util.c index fceebe2..f1b508c 100644 --- a/src/mpid/pamid/src/mpidi_util.c +++ b/src/mpid/pamid/src/mpidi_util.c @@ -1256,3 +1256,557 @@ int MPIDI_atoll(char* str_in, long long* val) } + +/****************************************************/ +/* Collective Selection Benchmarking utils */ +/****************************************************/ + +const char * xfer_array_str[PAMI_XFER_COUNT]; +int task_id; +int num_tasks; + +static int MPIDI_collsel_print_usage() +{ + if(!task_id) + fputs("Usage: pami_tune [options]\n\ +Options:\n\ + -c Comma separated list of collectives to benchmark\n\ + Valid options are: \n\ + allgather, allgatherv, allgatherv_int, allreduce, alltoall,\n\ + alltoallv, alltoallv_int, ambroadcast, amgather, amreduce,\n\ + amscatter, barrier, broadcast, gather, gatherv, gatherv_int,\n\ + reduce, reduce_scatter, scan, scatter, scatterv, scatterv_int\n\ + (Default: all collectives)\n\n\ + -m Comma separated list of message sizes to benchmark\n\ + (Default: 1 to 2^k, where k <= 20)\n\n\ + -g Comma separated list of geometry sizes to benchmark\n\ + (Default: 2 to 2^k, where k <= world geometry size)\n\n\ + -i Number of benchmark iterations per algorithm\n\ + (Default: 100)\n\n\ + -f <file> Input file containing benchmark parameters\n\ + You can override a parameter with a command line argument\n\n\ + -o <file> Output XML file containing benchmark results\n\ + (Default: pami_tune_results.xml)\n\n\ + -d Diagnostics mode. Verify correctness of collective algorithms\n\ + (Default: Disabled)\n\n\ + -v Verbose mode\n\ + (Default: Disabled)\n\n\ + -x Checkpoint mode. Enable pami_tune checkpointing\n\ + (Default: Disabled)\n\n\ + -h Print this help message\n\n", stdout); + + return 0; +} + + +static void MPIDI_collsel_init_advisor_params(advisor_params_t *params) +{ + params->num_collectives = 0; + params->num_procs_per_node = 0; + params->num_geometry_sizes = 0; + params->num_message_sizes = 0; + params->collectives = NULL; + params->procs_per_node = NULL; + params->geometry_sizes = NULL; + params->message_sizes = NULL; + params->iter = 100; + /* Set the following to -1, so that we can + check if the user has set them or not */ + params->verify = -1; + params->verbose = -1; + params->checkpoint = -1; +} + + +static void MPIDI_collsel_free_advisor_params(advisor_params_t *params) +{ + if(params->collectives) MPIU_Free(params->collectives); + if(params->procs_per_node) MPIU_Free(params->procs_per_node); + if(params->geometry_sizes) MPIU_Free(params->geometry_sizes); + if(params->message_sizes) MPIU_Free(params->message_sizes); +} + + +static void MPIDI_collsel_init_xfer_tables() +{ + xfer_array_str[PAMI_XFER_BROADCAST] ="broadcast"; + xfer_array_str[PAMI_XFER_ALLREDUCE] ="allreduce"; + xfer_array_str[PAMI_XFER_REDUCE] ="reduce"; + xfer_array_str[PAMI_XFER_ALLGATHER] ="allgather"; + xfer_array_str[PAMI_XFER_ALLGATHERV] ="allgatherv"; + xfer_array_str[PAMI_XFER_ALLGATHERV_INT]="allgatherv_int"; + xfer_array_str[PAMI_XFER_SCATTER] ="scatter"; + xfer_array_str[PAMI_XFER_SCATTERV] ="scatterv"; + xfer_array_str[PAMI_XFER_SCATTERV_INT] ="scatterv_int"; + xfer_array_str[PAMI_XFER_GATHER] ="gather"; + xfer_array_str[PAMI_XFER_GATHERV] ="gatherv"; + xfer_array_str[PAMI_XFER_GATHERV_INT] ="gatherv_int"; + xfer_array_str[PAMI_XFER_BARRIER] ="barrier"; + xfer_array_str[PAMI_XFER_ALLTOALL] ="alltoall"; + xfer_array_str[PAMI_XFER_ALLTOALLV] ="alltoallv"; + xfer_array_str[PAMI_XFER_ALLTOALLV_INT] ="alltoallv_int"; + xfer_array_str[PAMI_XFER_SCAN] ="scan"; + xfer_array_str[PAMI_XFER_REDUCE_SCATTER]="reduce_scatter"; + xfer_array_str[PAMI_XFER_AMBROADCAST] ="ambroadcast"; + xfer_array_str[PAMI_XFER_AMSCATTER] ="amscatter"; + xfer_array_str[PAMI_XFER_AMGATHER] ="amgather"; + xfer_array_str[PAMI_XFER_AMREDUCE] ="amreduce"; +} + +static int MPIDI_collsel_process_collectives(char *coll_arg, advisor_params_t *params) +{ + int i, ret = 0, arg_len = strlen(coll_arg); + char *collectives = (char *) MPIU_Malloc(arg_len + 1); + char *coll; + /* if already set via config file, free it */ + if(params->collectives) + { + MPIU_Free(params->collectives); + params->num_collectives = 0; + } + /* Allocating some extra space should be fine */ + params->collectives = (pami_xfer_type_t *)MPIU_Malloc(sizeof(pami_xfer_type_t)*PAMI_XFER_COUNT); + + strcpy(collectives, coll_arg); + coll = strtok(collectives,","); + while (coll != NULL) + { + for(i=0; i<PAMI_XFER_COUNT; i++) + { + if(strcmp(coll, xfer_array_str[i]) == 0) + { + params->collectives[params->num_collectives++] = i; + break; + } + } + /* arg did not match any collective */ + if(i == PAMI_XFER_COUNT) + { + MPIU_Free(params->collectives); + params->collectives = NULL; + if(!task_id) + { + fprintf(stderr, "Invalid collective: %s\n", coll); + } + ret = 1; + break; + } + coll = strtok(NULL,","); + } + MPIU_Free(collectives); + return ret; +} + + +static int MPIDI_collsel_process_msg_sizes(char *msg_sizes_arg, advisor_params_t *params) +{ + int ret = 0, arg_len = strlen(msg_sizes_arg); + char *msg_sizes = (char *) MPIU_Malloc(arg_len + 1); + char *msg_sz; + size_t tmp; + /* if already set via config file, free it */ + if(params->message_sizes) + { + MPIU_Free(params->message_sizes); + params->num_message_sizes = 0; + } + /* Allocating some extra space should be fine */ + params->message_sizes = (size_t *)MPIU_Malloc(sizeof(size_t) * 50); + + strcpy(msg_sizes, msg_sizes_arg); + msg_sz = strtok(msg_sizes,","); + while (msg_sz != NULL) + { + tmp = strtol(msg_sz, NULL, 10); + if(tmp == 0) + { + MPIU_Free(params->message_sizes); + params->message_sizes = NULL; + if(!task_id) + { + fprintf(stderr, "Invalid message size: %s\n", msg_sz); + } + ret = 1; + break; + } + + params->message_sizes[params->num_message_sizes++] = tmp; + msg_sz = strtok(NULL,","); + } + MPIU_Free(msg_sizes); + return ret; +} + + +static int MPIDI_collsel_process_geo_sizes(char *geo_sizes_arg, advisor_params_t *params) +{ + int ret = 0, arg_len = strlen(geo_sizes_arg); + char *geo_sizes = (char *) MPIU_Malloc(arg_len + 1); + char *geo_sz; + size_t tmp; + /* if already set via config file, free it */ + if(params->geometry_sizes) + { + MPIU_Free(params->geometry_sizes); + params->num_geometry_sizes = 0; + } + /* Allocating some extra space should be fine */ + params->geometry_sizes = (size_t *)MPIU_Malloc(sizeof(size_t) * 50); + + strcpy(geo_sizes, geo_sizes_arg); + geo_sz = strtok(geo_sizes,","); + while (geo_sz != NULL) + { + tmp = strtol(geo_sz, NULL, 10); + if(tmp < 2 || tmp > num_tasks) + { + MPIU_Free(params->geometry_sizes); + params->geometry_sizes = NULL; + if(!task_id) + { + fprintf(stderr, "Invalid geometry size: %s\n", geo_sz); + } + ret = 1; + break; + } + + params->geometry_sizes[params->num_geometry_sizes++] = tmp; + geo_sz = strtok(NULL,","); + } + MPIU_Free(geo_sizes); + return ret; +} + + +static int MPIDI_collsel_process_output_file(char *filename, char **out_file) +{ + char *newname; + int i, filename_len, ret = 0; + + filename_len = strlen(filename); + + /* Check if file already exists */ + if(access(filename, F_OK) == 0) + { + fprintf(stderr, "File %s already exists, renaming existing file\n", filename); + newname = (char *) MPIU_Malloc(filename_len + 4); + for (i = 0; i < 500; ++i) + { + sprintf(newname,"%s.%d", filename, i); + if(!(access(newname, F_OK) == 0)) + { + ret = rename(filename, newname); + break; + } + } + MPIU_Free(newname); + if(i == 500 || ret != 0) + { + fprintf(stderr, "Error renaming file\n"); + return 1; + } + } + /* if file name is already set via config file, free it */ + if(*out_file) MPIU_Free(*out_file); + *out_file = (char *)MPIU_Malloc(filename_len + 1); + strcpy(*out_file, filename); + + return ret; +} + + +static char* MPIDI_collsel_ltrim(char *line) +{ + while(*line && isspace(*line)) + ++line; + + return line; +} + +static char* MPIDI_collsel_rtrim(char *line) +{ + char *end = line + strlen(line); + while(end > line && isspace(*--end)) + *end = '\0'; + + return line; +} + +static int MPIDI_collsel_checkvalue(char *name, char *value, const char *filename, int *ival) +{ + int ret = 0; + char *tmp; + *ival = (int)strtol(value, &tmp, 10); + if(*ival != 1) + { + if((*ival == 0 && value == tmp)|| *ival != 0) + { + if(!task_id) + fprintf(stderr, "Invalid value for %s parameter: %s in file: %s\n", name, value, filename); + ret = 1; + } + } + return ret; +} + + +static int MPIDI_collsel_process_ini_file(const char *filename, advisor_params_t *params, char **out_file) +{ + char *line, *start, *name, *value; + int ret = 0; + FILE *file = fopen(filename, "r"); + if(!file) + { + fprintf(stderr, "Error. Can't open file %s\n", filename); + return 1; + } + line = (char *) MPIU_Malloc(1000); + + while (fgets(line, 1000, file) != NULL) + { + start = MPIDI_collsel_ltrim(MPIDI_collsel_rtrim(line)); + /* Skip comments and sections */ + if(*start == ';' || *start == '[' || *start == '#') + continue; + name = strtok(line, "="); + if(name == NULL) continue; + value = strtok(NULL, "="); + if(value == NULL) continue; + name = MPIDI_collsel_rtrim(name); + value = MPIDI_collsel_ltrim(value); + /* Do not override command line values if they are set */ + if(strcmp(name, "collectives") == 0) + { + if(!params->collectives) + ret = MPIDI_collsel_process_collectives(value, params); + } + else if(strcmp(name, "message_sizes") == 0) + { + if(!params->message_sizes) + ret = MPIDI_collsel_process_msg_sizes(value, params); + } + else if(strcmp(name, "geometry_sizes") == 0) + { + if(!params->geometry_sizes) + ret = MPIDI_collsel_process_geo_sizes(value, params); + } + else if(strcmp(name, "output_file") == 0) + { + if(!*out_file && !task_id) /* Only task 0 creates o/p file */ + ret = MPIDI_collsel_process_output_file(value, out_file); + } + else if(strcmp(name, "iterations") == 0) + { + if(params->iter == 100) + { + params->iter = atoi(value); + if(params->iter <= 0) + { + if(!task_id) + fprintf(stderr, "Invalid iteration count: %s in file: %s\n", value, filename); + ret = 1; + } + } + } + else if(strcmp(name, "verbose") == 0) + { + if(params->verbose == -1) + ret = MPIDI_collsel_checkvalue(name, value, filename, &params->verbose); + } + else if(strcmp(name, "diagnostics") == 0) + { + if(params->verify == -1) + ret = MPIDI_collsel_checkvalue(name, value, filename, &params->verify); + } + else if(strcmp(name, "checkpoint") == 0) + { + if(params->checkpoint == -1) + ret = MPIDI_collsel_checkvalue(name, value, filename, &params->checkpoint); + } + else + { + fprintf(stderr, "Invalid parameter: %s in file: %s\n", name, filename); + ret = 1; + } + if(ret) break; + } + MPIU_Free(line); + fclose(file); + + return ret; +} + + +static int MPIDI_collsel_process_arg(int argc, char *argv[], advisor_params_t *params, char ** out_file) +{ + int i,c,ret = 0; + + MPIDI_collsel_init_xfer_tables(); + params->verify = 0; + + opterr = 0; + while ((c = getopt (argc, argv, "c:m:g:f:o:i:v::d::x::h::")) != -1) + { + switch (c) + { + case 'c': + ret = MPIDI_collsel_process_collectives(optarg, params); + break; + case 'm': + ret = MPIDI_collsel_process_msg_sizes(optarg, params); + break; + case 'g': + ret = MPIDI_collsel_process_geo_sizes(optarg, params); + break; + case 'f': + ret = MPIDI_collsel_process_ini_file(optarg, params, out_file); + break; + case 'o': + if(!task_id) /* Only task 0 creates o/p file */ + ret = MPIDI_collsel_process_output_file(optarg, out_file); + break; + case 'i': + params->iter = atoi(optarg); + if(params->iter <= 0) + { + if(!task_id) + fprintf(stderr, "Invalid iteration count %s\n", optarg); + ret = 1; + } + break; + case 'd': + params->verify = 1; + break; + case 'v': + params->verbose = 1; + break; + case 'x': + params->checkpoint = 1; + break; + case 'h': + ret = 2; + break; + case '?': + if(!task_id) + { + if (optopt == 'c' || optopt == 'm' || optopt == 'g' || + optopt == 'f' || optopt == 'o' || optopt == 'i') + { + fprintf (stderr, "Option -%c requires an argument.\n", optopt); + } + else if (isprint (optopt)) + { + fprintf (stderr, "Unknown option `-%c'.\n", optopt); + } + else + { + fprintf (stderr, "Unknown option character `\\x%x'.\n", optopt); + } + } + ret = 1; + break; + default: + abort(); + break; + } + if(ret) return ret; + } + if(!task_id) + { + if (optind < argc) + { + printf ("Non-option arguments: "); + while (optind < argc) + printf ("%s ", argv[optind++]); + printf ("\n"); + } + } + + /* If user did not specify any collectives, benchmark all */ + if(params->num_collectives == 0) + { + params->collectives = (pami_xfer_type_t *)MPIU_Malloc(sizeof(pami_xfer_type_t)*PAMI_XFER_COUNT); + for(i = 0; i < PAMI_XFER_COUNT; i++) + { + params->collectives[params->num_collectives++] = i; + } + } + /* If user did not set any of the following parameters, disable them */ + if(params->verbose == -1) params->verbose = 0; + if(params->verify == -1) params->verify = 0; + if(params->checkpoint == -1) params->checkpoint = 0; + return 0; +} + + +static void MPIDI_collsel_print_params(advisor_params_t *params, char *output_file) +{ + size_t i; + printf(" Benchmark Parameters\n"); + printf(" --------------------\n"); + + printf(" Collectives:\n "); + for(i=0; i<params->num_collectives; i++){ + printf(" %s", xfer_array_str[params->collectives[i]]); + } + printf("\n Geometry sizes:\n "); + for(i=0; i<params->num_geometry_sizes; i++){ + printf(" %zu", params->geometry_sizes[i]); + } + printf("\n Message sizes:\n "); + for(i=0; i<params->num_message_sizes; i++){ + printf(" %zu", params->message_sizes[i]); + } + printf("\n Iterations : %d\n", params->iter); + printf(" Output file : %s\n", output_file); + printf(" Checkpoint mode : %d\n", params->checkpoint); + printf(" Diagnostics mode : %d\n", params->verify); + printf(" Verbose mode : %d\n", params->verbose); +} + + +int MPIDI_collsel_pami_tune_parse_params(int argc, char ** argv) +{ + pami_configuration_t config; + MPIDI_Collsel_output_file = NULL; + num_tasks = PAMIX_Client_query(MPIDI_Client, PAMI_CLIENT_NUM_TASKS).value.intval; + task_id = PAMIX_Client_query(MPIDI_Client, PAMI_CLIENT_TASK_ID ).value.intval; + MPIDI_collsel_init_advisor_params(&MPIDI_Collsel_advisor_params); + if(num_tasks < 2) + { + fprintf (stderr, "Error: pami_tune must be run with more than one task\n"); + fflush(stderr); + MPIDI_collsel_print_usage(); + return PAMI_ERROR; + } + MPIDI_Collsel_advisor_params.procs_per_node = (size_t*)MPIU_Malloc(1 * sizeof(size_t)); + MPIDI_Collsel_advisor_params.num_procs_per_node = 1; + MPIDI_Collsel_advisor_params.procs_per_node[0] = PAMIX_Client_query(MPIDI_Client, PAMI_CLIENT_NUM_LOCAL_TASKS ).value.intval; + + int result = MPIDI_collsel_process_arg(argc, argv, &MPIDI_Collsel_advisor_params, &MPIDI_Collsel_output_file); + if(result) + { + MPIDI_collsel_print_usage(); + return PAMI_ERROR; + } + + if(MPIDI_Collsel_output_file == NULL && !task_id) + { + if(MPIDI_collsel_process_output_file("pami_tune_results.xml", &MPIDI_Collsel_output_file)) + { + return PAMI_ERROR; + } + } + if(MPIDI_Collsel_advisor_params.verbose && !task_id) + MPIDI_collsel_print_params(&MPIDI_Collsel_advisor_params, MPIDI_Collsel_output_file); + + return PAMI_SUCCESS; + +} + +void MPIDI_collsel_pami_tune_cleanup() +{ + MPIDI_collsel_free_advisor_params(&MPIDI_Collsel_advisor_params); +} + + http://git.mpich.org/mpich.git/commitdiff/5673b82ca24c7481e312dc7573c84811c… commit 5673b82ca24c7481e312dc7573c84811c8244bb9 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Mon May 6 18:37:54 2013 -0500 Initial drafts of MPID_Win_set/get_info in pamid. No reviewer. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/Makefile.mk b/src/mpid/pamid/src/onesided/Makefile.mk index 329da4a..f068454 100644 --- a/src/mpid/pamid/src/onesided/Makefile.mk +++ b/src/mpid/pamid/src/onesided/Makefile.mk @@ -35,6 +35,8 @@ lib_lib@MPILIBNAME@_la_SOURCES += \ src/mpid/pamid/src/onesided/mpid_win_lock.c \ src/mpid/pamid/src/onesided/mpid_win_pscw.c \ src/mpid/pamid/src/onesided/mpid_win_put.c \ + src/mpid/pamid/src/onesided/mpid_win_get_info.c \ + src/mpid/pamid/src/onesided/mpid_win_set_info.c \ src/mpid/pamid/src/onesided/mpidi_win_control.c diff --git a/src/mpid/pamid/src/onesided/mpid_win_get_info.c b/src/mpid/pamid/src/onesided/mpid_win_get_info.c new file mode 100644 index 0000000..ba16658 --- /dev/null +++ b/src/mpid/pamid/src/onesided/mpid_win_get_info.c @@ -0,0 +1,50 @@ +/* begin_generated_IBM_copyright_prolog */ +/* */ +/* This is an automatically generated copyright prolog. */ +/* After initializing, DO NOT MODIFY OR MOVE */ +/* --------------------------------------------------------------- */ +/* Licensed Materials - Property of IBM */ +/* Blue Gene/Q 5765-PER 5765-PRP */ +/* */ +/* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */ +/* US Government Users Restricted Rights - */ +/* Use, duplication, or disclosure restricted */ +/* by GSA ADP Schedule Contract with IBM Corp. */ +/* */ +/* --------------------------------------------------------------- */ +/* */ +/* end_generated_IBM_copyright_prolog */ +/* (C)Copyright IBM Corp. 2007, 2011 */ +/** + * \file src/onesided/mpid_win_get_info.c + * \brief ??? + */ +#include "mpidi_onesided.h" + +/** + * \brief MPI-PAMI glue for MPI_WIN_GET_INFO function + * + * \param[in] win Window + * \param[in] info_p_p Info hint + * \return MPI_SUCCESS + */ +#undef FUNCNAME +#define FUNCNAME MPID_Win_get_info +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +int +MPID_Win_get_info(MPID_Win *win, + MPID_Info **info_p_p) +{ + int mpi_errno = MPI_SUCCESS; + + /* Allocate an empty info object */ + mpi_errno = MPIU_Info_alloc(info_p_p); + if (mpi_errno != MPI_SUCCESS) + goto fn_fail; + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} diff --git a/src/mpid/pamid/src/onesided/mpid_win_set_info.c b/src/mpid/pamid/src/onesided/mpid_win_set_info.c new file mode 100644 index 0000000..b56a43a --- /dev/null +++ b/src/mpid/pamid/src/onesided/mpid_win_set_info.c @@ -0,0 +1,40 @@ +/* begin_generated_IBM_copyright_prolog */ +/* */ +/* This is an automatically generated copyright prolog. */ +/* After initializing, DO NOT MODIFY OR MOVE */ +/* --------------------------------------------------------------- */ +/* Licensed Materials - Property of IBM */ +/* Blue Gene/Q 5765-PER 5765-PRP */ +/* */ +/* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */ +/* US Government Users Restricted Rights - */ +/* Use, duplication, or disclosure restricted */ +/* by GSA ADP Schedule Contract with IBM Corp. */ +/* */ +/* --------------------------------------------------------------- */ +/* */ +/* end_generated_IBM_copyright_prolog */ +/* (C)Copyright IBM Corp. 2007, 2011 */ +/** + * \file src/onesided/mpid_win_set_info.c + * \brief ??? + */ +#include "mpidi_onesided.h" + +/** + * \brief MPI-PAMI glue for MPI_WIN_SET_INFO function + * + * \param[in] win Window + * \param[in] info Info hint + * \return MPI_SUCCESS + */ +#undef FUNCNAME +#define FUNCNAME MPID_Win_set_info +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +int +MPID_Win_set_info(MPID_Win *win, + MPID_Info *info) +{ + return MPI_SUCCESS; +} http://git.mpich.org/mpich.git/commitdiff/a28689d4ff7104ba525ddc5ea512e2d45… commit a28689d4ff7104ba525ddc5ea512e2d45116c1ce Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Tue May 7 11:18:41 2013 -0500 Rename mpich2 to mpich. Also fixes tt#1829. The IBM contributed parts of the patch were reviewed by Mike Blocksome @ IBM. diff --git a/src/mpid/pamid/include/mpix.h b/src/mpid/pamid/include/mpix.h index 2f1f345..8996d82 100644 --- a/src/mpid/pamid/include/mpix.h +++ b/src/mpid/pamid/include/mpix.h @@ -214,7 +214,7 @@ extern "C" { * The communicator is created to match the size of each dimension, the * physical coords on each node, and the torus/mesh link status. * - * Because of MPICH2 dimension ordering, the associated arrays (i.e. coords, + * Because of MPICH dimension ordering, the associated arrays (i.e. coords, * sizes, and periods) are in [a, b, c, d, e, t] order. Consequently, when * using the default ABCDET mapping, the rank in cart_comm will match the rank * in MPI_COMM_WORLD. However, when using a non-default mapping or a mapfile diff --git a/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c b/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c index 6e6c39d..68b818b 100644 --- a/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c +++ b/src/mpid/pamid/src/dyntask/mpid_comm_spawn_multiple.c @@ -108,7 +108,7 @@ int MPID_Comm_spawn_multiple(int count, char *array_of_commands[], int mpi_errno = MPI_SUCCESS; /* We allow an empty implementation of this function to - simplify building MPICH2 on systems that have difficulty + simplify building MPICH on systems that have difficulty supporing process creation */ mpi_errno = MPIDI_Comm_spawn_multiple(count, array_of_commands, array_of_argv, array_of_maxprocs, diff --git a/src/openpa/Makefile.am b/src/openpa/Makefile.am index fa3afba..b119b3b 100644 --- a/src/openpa/Makefile.am +++ b/src/openpa/Makefile.am @@ -22,7 +22,7 @@ prettycheck: | highlight --color=yellow 'WARN(ING)?' \ | highlight --color=cyan -- '-SKIP-' -# This target exists to smooth the integration with MPICH2's upcoming build +# This target exists to smooth the integration with MPICH's upcoming build # system changes to support parallel make (i.e. "make -j 4"). It serves no # other purpose, but should _not_ be removed unless you know what you are doing. all-executable: @@ -32,7 +32,7 @@ pkgconfigdir = @pkgconfigdir@ pkgconfig_DATA = openpa.pc $(pkgconfig_DATA): config.status -# make MPICH2 happy +# make MPICH happy coverage: .PHONY: prettycheck all-executable coverage diff --git a/src/openpa/README b/src/openpa/README index 410b071..f668e6b 100644 --- a/src/openpa/README +++ b/src/openpa/README @@ -6,7 +6,7 @@ library that provides atomic primitives (and related constructs) for high performance, concurrent software. This project is a collaboration between the Mathematics and Computer Science (MCS) division at Argonne National Laboratory (ANL) and the HDF Group. The code was originally -derived from work on the MPICH2 project. +derived from work on the MPICH project. Project documentation and bug tracking can be found at: diff --git a/src/openpa/autogen.sh b/src/openpa/autogen.sh index 8dfe3a5..f05c2f7 100755 --- a/src/openpa/autogen.sh +++ b/src/openpa/autogen.sh @@ -1,7 +1,7 @@ #!/bin/sh -if [ -n "$MPICH2_AUTOTOOLS_DIR" ] ; then - autoreconf=${MPICH2_AUTOTOOLS_DIR}/autoreconf +if [ -n "$MPICH_AUTOTOOLS_DIR" ] ; then + autoreconf=${MPICH_AUTOTOOLS_DIR}/autoreconf else autoreconf=${AUTORECONF:-autoreconf} fi diff --git a/src/openpa/configure.ac b/src/openpa/configure.ac index 11d677f..bf682fd 100644 --- a/src/openpa/configure.ac +++ b/src/openpa/configure.ac @@ -160,7 +160,7 @@ AC_CHECK_SIZEOF([void *]) AC_CHECK_SIZEOF([int]) dnl Check for __attribute__ support. This was originally taken from -dnl the PAC_C_GNU_ATTRIBUTE macro in mpich2. +dnl the PAC_C_GNU_ATTRIBUTE macro in mpich. dnl dnl We start by requiring Gcc. Some other compilers accept __attribute__ dnl but generate warning messages, or have different interpretations diff --git a/src/openpa/src/primitives/opa_by_lock.h b/src/openpa/src/primitives/opa_by_lock.h index 2d7cd28..ea5d18c 100644 --- a/src/openpa/src/primitives/opa_by_lock.h +++ b/src/openpa/src/primitives/opa_by_lock.h @@ -16,7 +16,7 @@ /* defined in opa_primitives.c */ extern pthread_mutex_t *OPA_emulation_lock; -/* FIXME these make less sense now that OPA is not inside of MPICH2. Is there a +/* FIXME these make less sense now that OPA is not inside of MPICH. Is there a simpler name/scheme that could be used here instead? [goodell@ 2009-02-19] */ #define OPA_IPC_SINGLE_CS_ENTER(msg) \ do { \ diff --git a/src/pm/smpd/README b/src/pm/smpd/README index 4650f22..ed64668 100644 --- a/src/pm/smpd/README +++ b/src/pm/smpd/README @@ -43,14 +43,14 @@ hosts=<host1 host2 host3 ...> log=<yes or no> logfile=<path/filename> -After mpich is built, the smpd commands are in mpich2/bin, or the bin +After mpich is built, the smpd commands are in mpich/bin, or the bin subdirectory of the install directory if you have done an install. You should put this (bin) directory in your PATH in your .cshrc or .bashrc: -Put in .cshrc: setenv PATH /home/you/mpich2/bin:$PATH +Put in .cshrc: setenv PATH /home/you/mpich/bin:$PATH -Put in .bashrc: export PATH=/home/you/mpich2/bin:$PATH +Put in .bashrc: export PATH=/home/you/mpich/bin:$PATH You can run something with mpiexec @@ -59,7 +59,7 @@ donner.mcs.anl.gov foo.mcs.anl.gov donner% -You can run an mpich2 job: +You can run an mpich job: donner% mpiexec -n 10 /home/ashton/hellow Hello world from process 0 of 10 diff --git a/src/pmi/pmi2/poe/subconfigure.m4 b/src/pmi/pmi2/poe/subconfigure.m4 index 1b10221..5e2ce7d 100644 --- a/src/pmi/pmi2/poe/subconfigure.m4 +++ b/src/pmi/pmi2/poe/subconfigure.m4 @@ -1,5 +1,5 @@ [#] start of __file__ -dnl MPICH2_SUBCFG_AFTER=src/pmi +dnl MPICH_SUBCFG_AFTER=src/pmi AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[ ]) diff --git a/src/pmi/pmi2/simple/subconfigure.m4 b/src/pmi/pmi2/simple/subconfigure.m4 index 530012a..1b0f5cc 100644 --- a/src/pmi/pmi2/simple/subconfigure.m4 +++ b/src/pmi/pmi2/simple/subconfigure.m4 @@ -1,5 +1,5 @@ [#] start of __file__ -dnl MPICH2_SUBCFG_AFTER=src/pmi +dnl MPICH_SUBCFG_AFTER=src/pmi AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[ ]) ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_constants.h | 1 + src/mpid/pamid/include/mpidi_datatypes.h | 131 +++++- src/mpid/pamid/include/mpidi_externs.h | 5 + src/mpid/pamid/include/mpidi_prototypes.h | 29 + src/mpid/pamid/include/mpidimpl.h | 14 + src/mpid/pamid/include/mpix.h | 2 +- .../pamid/src/coll/allgather/mpido_allgather.c | 57 ++- .../pamid/src/coll/allgatherv/mpido_allgatherv.c | 66 ++- .../pamid/src/coll/allreduce/mpido_allreduce.c | 47 ++- src/mpid/pamid/src/coll/alltoall/mpido_alltoall.c | 124 ++++- .../pamid/src/coll/alltoallv/mpido_alltoallv.c | 58 ++- src/mpid/pamid/src/coll/barrier/mpido_barrier.c | 14 + src/mpid/pamid/src/coll/bcast/mpido_bcast.c | 35 ++ src/mpid/pamid/src/coll/coll_utils.c | 134 +++++ src/mpid/pamid/src/coll/gather/mpido_gather.c | 133 ++++- src/mpid/pamid/src/coll/gatherv/mpido_gatherv.c | 200 ++++++-- src/mpid/pamid/src/coll/reduce/mpido_reduce.c | 44 ++- src/mpid/pamid/src/coll/scan/mpido_scan.c | 54 ++- src/mpid/pamid/src/coll/scatter/mpido_scatter.c | 164 +++++- src/mpid/pamid/src/coll/scatterv/mpido_scatterv.c | 191 ++++++-- src/mpid/pamid/src/comm/mpid_comm.c | 109 ++++ src/mpid/pamid/src/comm/mpid_selectcolls.c | 258 ++++++--- .../pamid/src/dyntask/mpid_comm_spawn_multiple.c | 2 +- src/mpid/pamid/src/mpid_init.c | 194 +++++++- src/mpid/pamid/src/mpidi_env.c | 35 +- src/mpid/pamid/src/mpidi_util.c | 568 ++++++++++++++++++++ src/mpid/pamid/src/onesided/Makefile.mk | 2 + .../mpid_mem.c => onesided/mpid_win_get_info.c} | 40 +- .../{mpid_probe.c => onesided/mpid_win_set_info.c} | 31 +- src/openpa/Makefile.am | 4 +- src/openpa/README | 2 +- src/openpa/autogen.sh | 4 +- src/openpa/configure.ac | 2 +- src/openpa/src/primitives/opa_by_lock.h | 2 +- src/pm/smpd/README | 8 +- src/pmi/pmi2/poe/subconfigure.m4 | 2 +- src/pmi/pmi2/simple/subconfigure.m4 | 2 +- 37 files changed, 2457 insertions(+), 311 deletions(-) copy src/mpid/pamid/src/{misc/mpid_mem.c => onesided/mpid_win_get_info.c} (68%) copy src/mpid/pamid/src/{mpid_probe.c => onesided/mpid_win_set_info.c} (73%) hooks/post-receive -- MPICH primary repository

1 0