This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 7228af25d5f7490f69c7929841cb7bc24d6d792e (commit) via e78f37339f9573de4158d4a404834955b8da42f0 (commit) via 6e7afae345cc450f7999adb6d52d1a1878c19504 (commit) via d3b99bb0be96a1721ced0b99f025796ca572d24d (commit) via fe43008c712fe7f33564e7d1179b9997087efe36 (commit) via d0c1c15acd0ed286c737570399e075f08b63e6fe (commit) via 709a3b3f17abf85d3055824d1a0bd6977e64b2f7 (commit) from 2a90c3644882f5ef805fed0f8add5c3ccbf3bd0e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/7228af25d5f7490f69c7929841cb7bc24d... commit 7228af25d5f7490f69c7929841cb7bc24d6d792e Author: Michael Blocksome <[email protected]> Date: Thu Apr 18 13:14:35 2013 -0500 Additional MPIX_IO_* functions and fortran interfaces. - add fortran interfaces for new MPIX_* functions - remove fortran interface for MPIX_Pset_io_node() - do not use the MPIX_Pset_io_node() function internally The MPIX_Pset_io_node() function has been deprecated. (ibm) CPS 92XKPE (ibm) 6c7f293ccda15d447369ee7327a225d3b7979f15 (ibm) 106eef011bc1e58ada7e208e8c56ee341045611b (ibm) 23b42d38b9f75ddd93701a021f84418a30a7693a (ibm) b39954ab0052e0c5676cd9c270f159a4f6a99828 Signed-off-by: Haizhu Liu <[email protected]> diff --git a/src/mpid/pamid/include/mpix.h b/src/mpid/pamid/include/mpix.h index 8996d82..6940ad9 100644 --- a/src/mpid/pamid/include/mpix.h +++ b/src/mpid/pamid/include/mpix.h @@ -78,6 +78,10 @@ extern "C" { /** * \brief Determine the rank-in-COMM_WORLD of the process associated with rank-in-comm + * + * Fortran interface: + * MPIX_COMM_RANK2GLOBAL (INTEGER comm, INTEGER crank, INTEGER grank, INTEGER ierr) + * * \param[in] comm The communicator associated with the input rank * \param[in] crank The rank-in-comm * \param[out] grank The rank-in-COMM_WORLD (AKA Global rank) @@ -139,6 +143,9 @@ extern "C" { * \brief Create a communicator such that all nodes in the same * communicator are served by the same I/O node * + * Fortran interface: + * MPIX_PSET_SAME_COMM_CREATE(INTEGER pset_comm, INTEGER ierr) + * * \note This is a collective operation on MPI_COMM_WORLD * * \param [out] pset_comm The new communicator @@ -151,6 +158,9 @@ extern "C" { * \brief Create a communicator such that all nodes in the same * communicator are served by a different I/O node * + * Fortran interface: + * MPIX_PSET_DIFF_COMM_CREATE(INTEGER pset_comm, INTEGER ierr) + * * \note This is a collective operation on MPI_COMM_WORLD * * \param [out] pset_comm The new communicator @@ -163,6 +173,9 @@ extern "C" { * \brief Create a communicator such that all nodes in the same * communicator are served by the same I/O node * + * Fortran interface: + * MPIX_PSET_SAME_COMM_CREATE (INTEGER parent_comm, INTEGER pset_comm, INTEGER ierr) + * * \note This is a collective operation on the parent communicator. * * \param [in] parent_comm The parent communicator @@ -176,6 +189,9 @@ extern "C" { * \brief Create a communicator such that all nodes in the same * communicator are served by a different I/O node * + * Fortran interface: + * MPIX_PSET_DIFF_COMM_CREATE (INTEGER parent_comm, INTEGER pset_comm, INTEGER ierr) + * * \note This is a collective operation on the parent communicator * * \param [in] parent_comm The parent communicator @@ -186,20 +202,65 @@ extern "C" { int MPIX_Pset_diff_comm_create_from_parent (MPI_Comm parent_comm, MPI_Comm *pset_comm); /** - * \brief Retrieve information about the I/O node associated with the - * local compute node. + * \brief Retrieve the identifier of the io node associated with the local compute node. + * + * The I/O node identifier is a unique number, yet it is not a monotonically + * increasing integer; such as a rank in a communicator. Multiple ranks, and + * multiple compute nodes, can be associated with the same I/O link. + * + * Fortran interface: + * MPIX_IO_NODE_ID (INTEGER io_node_id) + * + * \note On BG/Q two 'bridge' compute nodes are connected to each io node; + * each unique io node identifier will be associated with at most two + * different io link identifiers. + * + * \return I/O node identifier + */ + int MPIX_IO_node_id (); + + /** + * \brief Retrieve the identifier of the io link associated with the local compute node. * - * The I/O node route identifier is a unique number, yet it is not a - * monotonically increasing integer; such as a rank in a communicator. - * Multiple ranks, and multiple compute nodes, can be associated with the - * same I/O node route. + * The I/O link identifier is a unique number, yet it is not a monotonically + * increasing integer; such as a rank in a communicator. Multiple ranks, and + * multiple compute nodes, can be associated with the same I/O link. + * + * Fortran interface: + * MPIX_IO_LINK_ID (INTEGER io_link_id) + * + * \return I/O link identifier + */ + int MPIX_IO_link_id (); + + /** + * \brief Retrieve the number of hops to the io node associated with the local compute node. * * The distance to the I/O node is the number of hops on the torus from the - * local compute node to the associated I/O node. + * local compute node to the associated I/O node. The 'torus' vs 'mesh' + * attribute of each dimention is considered when determining the distance. + * + * Fortran interface: + * MPIX_IO_DISTANCE (INTEGER io_distance) * * \note On BG/Q the 'bridge' compute nodes are those nodes that are closest * to the I/O node and will have a distance of '1'. * + * \return number of hops to the I/O node + */ + int MPIX_IO_distance (); + + /** + * \brief Retrieve information about the I/O node associated with the + * local compute node. + * + * \deprecated This function has been replaced with MPIX_IO_link_id() and + * MPIX_IO_distance() + * + * \see MPIX_IO_link_id + * \see MPIX_IO_distance + * \see MPIX_IO_node_id + * * \param [out] io_node_route_id The unique I/O node route identifier * \param [out] distance_to_io_node The number of hops to the I/O node */ @@ -220,6 +281,9 @@ extern "C" { * in MPI_COMM_WORLD. However, when using a non-default mapping or a mapfile * the ranks will be different. * + * Fortran interface: + * MPIX_CART_COMM_CREATE (INTEGER cart_comm, INTEGER ierr) + * * \param [out] cart_comm The new Cartesian communicator * * \return MPI_SUCCESS or MPI_ERR_TOPOLOGY diff --git a/src/mpid/pamid/src/mpix/mpix.c b/src/mpid/pamid/src/mpix/mpix.c index 87741fc..88df322 100644 --- a/src/mpid/pamid/src/mpix/mpix.c +++ b/src/mpid/pamid/src/mpix/mpix.c @@ -520,17 +520,79 @@ MPIX_Get_last_algorithm_name(MPI_Comm comm, char *protocol, int length) } #undef FUNCNAME -#define FUNCNAME MPIX_Pset_ionode +#define FUNCNAME MPIX_IO_node_id #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -void -MPIX_Pset_io_node (int *io_node_route_id, int *distance_to_io_node) +int MPIX_IO_node_id () +{ + static unsigned long IO_node_id = ULONG_MAX; + + if (IO_node_id != ULONG_MAX) + return (int)(IO_node_id>>32); + + int rc; + int fd; + char* uci_str; + char buffer[4096]; + unsigned long uci; + + fd = open("/dev/bgpers", O_RDONLY, 0); + assert(fd>=0); + rc = read(fd, buffer, sizeof(buffer)); + assert(rc>0); + close(fd); + + uci_str = strstr(buffer, "BG_UCI="); + assert(uci_str); + uci_str += sizeof("BG_UCI=")-1; + + IO_node_id = strtoul(uci_str, NULL, 16); + return (int)(IO_node_id>>32); +} + +#undef FUNCNAME +#define FUNCNAME MPIX_IO_link_id +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +int MPIX_IO_link_id () +{ + int nA, nB, nC, nD, nE; /* Size of each torus dimension */ + int brA, brB, brC, brD, brE; /* The bridge node's coordinates */ + + Personality_t personality; + + Kernel_GetPersonality(&personality, sizeof(personality)); + + nA = personality.Network_Config.Anodes; + nB = personality.Network_Config.Bnodes; + nC = personality.Network_Config.Cnodes; + nD = personality.Network_Config.Dnodes; + nE = personality.Network_Config.Enodes; + + brA = personality.Network_Config.cnBridge_A; + brB = personality.Network_Config.cnBridge_B; + brC = personality.Network_Config.cnBridge_C; + brD = personality.Network_Config.cnBridge_D; + brE = personality.Network_Config.cnBridge_E; + + /* + * This is the bridge node, numbered in ABCDE order, E increments first. + * It is considered the unique "io node route identifer" because each + * bridge node only has one torus link to one io node. + */ + return brE + brD*nE + brC*nD*nE + brB*nC*nD*nE + brA*nB*nC*nD*nE; +}; + +#undef FUNCNAME +#define FUNCNAME MPIX_IO_distance +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +int MPIX_IO_distance () { int iA, iB, iC, iD, iE; /* The local node's coordinates */ int nA, nB, nC, nD, nE; /* Size of each torus dimension */ int brA, brB, brC, brD, brE; /* The bridge node's coordinates */ int Nflags; - int torusA, torusB, torusC, torusD, torusE; /* mesh == 0, torus == 1 */ int d1, d2; int dA, dB, dC, dD, dE; /* distance from local node to bridge node */ @@ -558,51 +620,59 @@ MPIX_Pset_io_node (int *io_node_route_id, int *distance_to_io_node) Nflags = personality.Network_Config.NetFlags; - if (Nflags & ND_ENABLE_TORUS_DIM_A) torusA = 1; - else torusA = 0; - if (Nflags & ND_ENABLE_TORUS_DIM_B) torusB = 1; - else torusB = 0; - if (Nflags & ND_ENABLE_TORUS_DIM_C) torusC = 1; - else torusC = 0; - if (Nflags & ND_ENABLE_TORUS_DIM_D) torusD = 1; - else torusD = 0; - if (Nflags & ND_ENABLE_TORUS_DIM_E) torusE = 1; - else torusE = 0; - - /* - * This is the bridge node, numbered in ABCDE order, E increments first. - * It is considered the unique "io node route identifer" because each - * bridge node only has one torus link to one io node. - */ - *io_node_route_id = brE + brD*nE + brC*nD*nE + brB*nC*nD*nE + brA*nB*nC*nD*nE; - - d1 = abs(iA - brA); - d2 = nA - d1; - if (torusA) dA = (d1 < d2) ? d1 : d2; - else dA = d1; + dA = abs(iA - brA); + if (Nflags & ND_ENABLE_TORUS_DIM_A) + { + d1 = dA; + d2 = nA - d1; + dA = (d1 < d2) ? d1 : d2; + } - d1 = abs(iB - brB); - d2 = nB - d1; - if (torusB) dB = (d1 < d2) ? d1 : d2; - else dB = d1; + dB = abs(iB - brB); + if (Nflags & ND_ENABLE_TORUS_DIM_B) + { + d1 = dB; + d2 = nB - d1; + dB = (d1 < d2) ? d1 : d2; + } - d1 = abs(iC - brC); - d2 = nC - d1; - if (torusC) dC = (d1 < d2) ? d1 : d2; - else dC = d1; + dC = abs(iC - brC); + if (Nflags & ND_ENABLE_TORUS_DIM_C) + { + d1 = dC; + d2 = nC - d1; + dC = (d1 < d2) ? d1 : d2; + } - d1 = abs(iD - brD); - d2 = nD - d1; - if (torusD) dD = (d1 < d2) ? d1 : d2; - else dD = d1; + dD = abs(iD - brD); + if (Nflags & ND_ENABLE_TORUS_DIM_D) + { + d1 = dD; + d2 = nD - d1; + dD = (d1 < d2) ? d1 : d2; + } - d1 = abs(iE - brE); - d2 = nE - d1; - if (torusE) dE = (d1 < d2) ? d1 : d2; - else dE = d1; + dE = abs(iE - brE); + if (Nflags & ND_ENABLE_TORUS_DIM_E) + { + d1 = dE; + d2 = nE - d1; + dE = (d1 < d2) ? d1 : d2; + } /* This is the number of hops to the io node */ - *distance_to_io_node = dA + dB + dC + dD + dE + 1; + return dA + dB + dC + dD + dE + 1; +}; + +#undef FUNCNAME +#define FUNCNAME MPIX_Pset_io_node +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +void +MPIX_Pset_io_node (int *io_node_route_id, int *distance_to_io_node) +{ + *io_node_route_id = MPIX_IO_link_id (); + *distance_to_io_node = MPIX_IO_distance (); return; }; @@ -622,7 +692,8 @@ int _MPIX_Pset_same_comm_create (MPID_Comm *parent_comm_ptr, MPID_Comm **pset_co int color, key; int mpi_errno; - MPIX_Pset_io_node (&color, &key); + color = MPIX_IO_link_id (); + key = MPIX_IO_distance (); /* * Use MPIR_Comm_split_impl to make a communicator of all ranks in the parent @@ -861,6 +932,105 @@ MPIX_Cart_comm_create (MPI_Comm *cart_comm) return MPI_SUCCESS; }; +/** + * \brief FORTRAN interface to MPIX_Comm_rank2global + * + * \param [in] comm Communicator + * \param [in] crank Pointer to the rank in the communicator variable + * \param [out] grank Pointer tot he global rank variable + * + * \return status + */ +int mpix_comm_rank2global (MPI_Comm *comm, int *crank, int *grank) +{ + return MPIX_Comm_rank2global (*comm, *crank, grank); +} + +/** + * \brief FORTRAN interface to MPIX_Pset_same_comm_create + * + * \param [out] pset_comm Communicator + * + * \return status + */ +int mpix_pset_same_comm_create (MPI_Comm *pset_comm) +{ + return MPIX_Pset_same_comm_create (pset_comm); +} + +/** + * \brief FORTRAN interface to MPIX_Pset_diff_comm_create + * + * \param [out] pset_comm Communicator + * + * \return status + */ +int mpix_pset_diff_comm_create (MPI_Comm *pset_comm) +{ + return MPIX_Pset_diff_comm_create (pset_comm); +} + +/** + * \brief FORTRAN interface to MPIX_Pset_same_comm_create_from_parent + * + * \param [in] parent_comm Parent communicator + * \param [out] pset_comm New pset communicator + * + * \return status + */ +int mpix_pset_same_comm_create_from_parent (MPI_Comm *parent_comm, MPI_Comm *pset_comm) +{ + return MPIX_Pset_same_comm_create_from_parent (*parent_comm, pset_comm); +} + +/** + * \brief FORTRAN interface to MPIX_Pset_diff_comm_create_from_parent + * + * \param [in] parent_comm Parent communicator + * \param [out] pset_comm New pset communicator + * + * \return status + */ +int mpix_pset_diff_comm_create_from_parent (MPI_Comm *parent_comm, MPI_Comm *pset_comm) +{ + return MPIX_Pset_diff_comm_create_from_parent (*parent_comm, pset_comm); +} + +/** + * \brief FORTRAN interface to MPIX_IO_node_id + * + * \param [out] io_node_id This rank's io node id + */ +void mpix_io_node_id (int *io_node_id) { *io_node_id = MPIX_IO_node_id(); } + +/** + * \brief FORTRAN interface to MPIX_IO_link_id + * + * \param [out] io_link_id This rank's io link id + */ +void mpix_io_link_id (int *io_link_id) { *io_link_id = MPIX_IO_link_id(); } + +/** + * \brief FORTRAN interface to MPIX_IO_distance + * + * \param [out] io_distance This rank's distance to the io node + */ +void mpix_io_distance (int *io_distance) { *io_distance = MPIX_IO_distance(); } + +/** + * \brief FORTRAN interface to MPIX_Cart_comm_create + * + * \param [out] cart_comm Communicator to create + * + * \return status + */ +int mpix_cart_comm_create (MPI_Comm *cart_comm) +{ + return MPIX_Cart_comm_create (cart_comm); +} + + + #endif #ifdef __PE__ http://git.mpich.org/mpich.git/commitdiff/e78f37339f9573de4158d4a404834955b8... commit e78f37339f9573de4158d4a404834955b8da42f0 Author: Michael Blocksome <[email protected]> Date: Thu Apr 18 11:33:27 2013 -0500 always disable eager for synchronous sends (ibm) CPS 95PHUZ (ibm) 7bfb869616ec9aaffe4e5de9e05840fed4e4065a (ibm) d5a3fc9e58ccfcf185f9671b06d8307fd56e653c Signed-off-by: Haizhu Liu <[email protected]> diff --git a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c index 805d310..8904ddd 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c @@ -439,6 +439,7 @@ if (!TOKEN_FLOW_CONTROL_ON) { #endif const unsigned isLocal = PAMIX_Task_is_local(dest_tid); + const unsigned data_sz_limit = isSync?UINT_MAX:data_sz; /* * Always use the short protocol when data_sz is small. @@ -456,7 +457,7 @@ if (!TOKEN_FLOW_CONTROL_ON) { /* * Use the eager protocol when data_sz is less than the eager limit. */ - else if (data_sz < MPIDI_PT2PT_EAGER_LIMIT(isInternal,isLocal)) + else if (data_sz_limit < MPIDI_PT2PT_EAGER_LIMIT(isInternal,isLocal)) { TRACE_ERR("Sending(eager%s%s) bytes=%u (eager_limit=%u)\n", isInternal==1?",internal":"", isLocal==1?",intranode":"", data_sz, MPIDI_PT2PT_EAGER_LIMIT(isInternal,isLocal)); MPIDI_SendMsg_eager(context, http://git.mpich.org/mpich.git/commitdiff/6e7afae345cc450f7999adb6d52d1a1878... commit 6e7afae345cc450f7999adb6d52d1a1878c19504 Author: Su Huang <[email protected]> Date: Mon Jun 3 11:09:47 2013 -0400 negative source returned from MPI_Waitany (ibm) D190899 (ibm) 6693a84bd469820d94e694bbb7e3fbe2b6188e2f Signed-off-by: Michael Blocksome <[email protected]> diff --git a/src/mpid/pamid/src/pt2pt/mpidi_done.c b/src/mpid/pamid/src/pt2pt/mpidi_done.c index 8e68aa9..9f2e062 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_done.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_done.c @@ -216,6 +216,9 @@ void MPIDI_Recvq_process_out_of_order_msgs(pami_task_t src, pami_context_t conte } else { MPIDI_RecvMsg_Unexp(ooreq, rreq->mpid.userbuf, rreq->mpid.userbufcount, rreq->mpid.datatype); rreq->status.count = ooreq->status.count; + rreq->status.MPI_SOURCE = ooreq->status.MPI_SOURCE; + rreq->status.MPI_TAG = ooreq->status.MPI_TAG; + rreq->mpid.envelope.msginfo.MPIseqno = ooreq->mpid.envelope.msginfo.MPIseqno; MPIDI_Request_complete(rreq); } MPID_Request_release(ooreq); http://git.mpich.org/mpich.git/commitdiff/d3b99bb0be96a1721ced0b99f025796ca5... commit d3b99bb0be96a1721ced0b99f025796ca572d24d Author: Su Huang <[email protected]> Date: Wed May 29 09:18:21 2013 -0400 pamid updates to handling trace buffers (ibm) D190239 (ibm) 83363688c21e16bf6f504d5d1bd28075a099bb89 Signed-off-by: Michael Blocksome <[email protected]> diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 62adcbf..fb56e3c 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -35,6 +35,7 @@ #include "mpidi_constants.h" #include "mpidi_platform.h" #include "pami.h" +#include "mpidi_trace.h" #if (MPIU_HANDLE_ALLOCATION_METHOD == MPIU_HANDLE_ALLOCATION_THREAD_LOCAL) && defined(__BGQ__) struct MPID_Request; diff --git a/src/mpid/pamid/include/mpidi_trace.h b/src/mpid/pamid/include/mpidi_trace.h index 9f04067..97f6b96 100644 --- a/src/mpid/pamid/include/mpidi_trace.h +++ b/src/mpid/pamid/include/mpidi_trace.h @@ -31,6 +31,11 @@ typedef struct { int rtag; /* tag of a received message */ int rlen; /* len of a received message */ int rctx; /* context of a received message */ + void * matchedHandle; /* a message with multiple handles */ + union { + uint flags; + struct { +#ifdef __BIG_ENDIAN__ uint posted:1; /* has the receive posted */ uint rzv:1; /* rendezvous message ? */ uint sync:1; /* synchronous message? */ @@ -49,8 +54,32 @@ typedef struct { uint matchedInWait:1;/* found a match in MPI_Wait() etc. */ uint ReadySend:1; /* a ready send messsage */ uint persist:1; /* persist communication */ - uint reserve:9; - void * matchedHandle; /* a message with multiple handles */ + uint reserve:1; + uint reserve1:8; +#else + uint reserve1:8; + uint reserve:1; + uint persist:1; /* persist communication */ + uint ReadySend:1; /* a ready send messsage */ + uint matchedInWait:1;/* found a match in MPI_Wait() etc. */ + uint matchedInUQ2:2;/* found a match in unexpected queue */ + uint matchedInUQ:2; /* found a match in unexpected queue */ + uint matchedInComp:1;/* found a match in completion handler */ + uint matchedInHH:1; /* found a match in header haldner */ + uint sync_com_in_HH:1; /* sync msg completed in header handler*/ + uint comp_in_HHV_noMatch:1;/* no matched in header handler EA */ + uint comp_in_HH:4; /* the msg completed in header handler */ + uint matchedInOOL:1;/* found a match in out of order list */ + uint ool:1; /* the msg arrived out of order */ + uint HH:1; /* header handler */ + uint sendFin:1; /* send complete info? */ + uint sendAck:1; /* send ack? */ + uint sync:1; /* synchronous message? */ + uint rzv:1; /* rendezvous message ? */ + uint posted:1; /* has the receive posted */ +#endif + }f; + }fl; } recv_status; typedef struct { @@ -64,6 +93,10 @@ typedef struct { unsigned short dummy; int tag; /* tag of a message */ int len; /* lengh of a message */ + union { + uint flags; + struct { +#ifdef __BIG_ENDIAN__ uint blocking:1; /* blocking send ? */ uint sync:1; /* sync message */ uint sendEnvelop:1; /* envelop send? */ @@ -80,7 +113,30 @@ typedef struct { uint ReadySend:1; /* ready send */ uint reqXfer:1; /* request message transfer */ uint persist:1; /* persistent communiation */ - uint reserved:15; + uint reserve:5; + uint reserve1:8; +#else + uint reserve1:8; + uint reserve:5; + uint persist:1; /* persistent communiation */ + uint reqXfer:1; /* request message transfer */ + uint ReadySend:1; /* ready send */ + uint complSync:1; /* complete sync */ + uint recvFin:1; /* recv complete information */ + uint recvAck:1; /* recv an ack from the receiver */ + uint sendComp:1; /* send complete */ + uint NoComp:4; /* no completion handler */ + uint use_pami_get:1;/* use only PAMI_Get() */ + uint memRegion:1; /* memory is registered */ + uint sendRzv:1; /* send via renzdvous protocol */ + uint sendEager:1; /* eager send */ + uint sendShort:1; /* send immediate */ + uint sendEnvelop:1; /* envelop send? */ + uint sync:1; /* sync message */ + uint blocking:1; /* blocking send ? */ +#endif + }f; + }fl; } send_status; typedef struct { @@ -94,15 +150,46 @@ typedef struct { int len; /* length of a receive message */ uint nMsgs; /* no. of messages have been received */ uint msgid; /* msg seqno of the matched message */ +#ifdef __BIG_ENDIAN__ uint sendCtx:16; /* context of incoming msg */ uint recvCtx:16; /* context of a posted receive */ +#else + uint recvCtx:16; /* context of a posted receive */ + uint sendCtx:16; /* context of incoming msg */ +#endif + union { + uint flags; + struct { +#ifdef __BIG_ENDIAN__ uint lw:4; /* use lw protocol immediate send */ uint persist:4; /* persistent communication */ uint blocking:2; /* blocking receive */ - uint reserve:22; + uint reserve:6; + uint reserve1:16; +#else + uint reserve1:16; + uint reserve:6; + uint blocking:2; /* blocking receive */ + uint persist:4; /* persistent communication */ + uint lw:4; /* use lw protocol immediate send */ +#endif + }f; + }fl; } posted_recv; -#define MPIDI_SET_PR_REC(rreq,buf,ct,ll,dt,pami_id,rank,tag,comm,is_blk) { \ + +typedef struct MPIDI_Trace_buf { + recv_status *R; /* record incoming messages */ + posted_recv *PR; /* record posted receive */ + send_status *S; /* send messages */ + int totPR; /* total no. of poste receive */ +} MPIDI_Trace_buf_t; + +MPIDI_Trace_buf_t *MPIDI_Trace_buf; + + + +#define MPIDI_SET_PR_REC(rreq,buf,ct,dt,pami_id,rank,tag,comm,is_blk) { \ int idx,src,seqNo,x; \ if (pami_id != MPI_ANY_SOURCE) \ src=pami_id; \ @@ -118,42 +205,53 @@ typedef struct { MPIDI_Trace_buf[src].PR[idx].bufadd = buf; \ MPIDI_Trace_buf[src].PR[idx].msgid = seqNo; \ MPIDI_Trace_buf[src].PR[idx].count = ct; \ - MPIDI_Trace_buf[src].PR[idx].len = ll; \ MPIDI_Trace_buf[src].PR[idx].datatype = dt; \ MPIDI_Trace_buf[src].PR[idx].tag=tag; \ MPIDI_Trace_buf[src].PR[idx].sendCtx=comm->context_id; \ MPIDI_Trace_buf[src].PR[idx].recvCtx=comm->recvcontext_id;\ - MPIDI_Trace_buf[src].PR[idx].blocking=is_blk; \ + MPIDI_Trace_buf[src].PR[idx].fl.f.blocking=is_blk; \ rreq->mpid.PR_idx=idx; \ } -#define MPIDI_GET_S_REC(sreq,ctx,isSync,dataSize) { \ +#define MPIDI_GET_S_REC(dd,sreq,ctx,isSync,dataSize) { \ send_status *sstatus; \ - int dest=sreq->mpid.partner_id; \ int seqNo=sreq->mpid.envelope.msginfo.MPIseqno; \ int idx = (seqNo & SEQMASK); \ - memset(&MPIDI_Trace_buf[dest].S[idx],0,sizeof(send_status));\ - sstatus=&MPIDI_Trace_buf[dest].S[idx]; \ + sreq->mpid.partner_id=dd; \ + memset(&MPIDI_Trace_buf[dd].S[idx],0,sizeof(send_status));\ + sstatus=&MPIDI_Trace_buf[dd].S[idx]; \ sstatus->req = (void *)sreq; \ sstatus->tag = sreq->mpid.envelope.msginfo.MPItag; \ sstatus->dest = sreq->mpid.peer_pami; \ sstatus->rank = sreq->mpid.peer_comm; \ sstatus->msgid = seqNo; \ - sstatus->sync = isSync; \ + sstatus->fl.f.sync = isSync; \ sstatus->sctx = ctx; \ sstatus->tag = sreq->mpid.envelope.msginfo.MPItag; \ sstatus->len= dataSize; \ sreq->mpid.idx=idx; \ } -typedef struct MPIDI_Trace_buf { - recv_status *R; /* record incoming messages */ - posted_recv *PR; /* record posted receive */ - send_status *S; /* send messages */ - int totPR; /* total no. of poste receive */ -} MPIDI_Trace_buf_t; -MPIDI_Trace_buf_t *MPIDI_Trace_buf; +#define TRACE_SET_S_BIT(dd,ii,mbr) MPIDI_Trace_buf[(dd)].S[(ii)].mbr=1; +#define TRACE_SET_R_BIT(dd,ii,mbr) MPIDI_Trace_buf[(dd)].R[(ii)].mbr=1; +#define TRACE_SET_S_VAL(dd,ii,mbr,val) MPIDI_Trace_buf[(dd)].S[(ii)].mbr=val; +#define TRACE_SET_R_VAL(dd,ii,mbr,val) MPIDI_Trace_buf[(dd)].R[(ii)].mbr=val; +#define TRACE_SET_REQ_VAL(ww,val1) ww=val1; +#define TRACE_MEMSET_R(tt,nbr,str) (memset(&MPIDI_Trace_buf[tt].R[(nbr & SEQMASK)],0,sizeof(str))); +#define TRACE_MEMSET_S(tt,nbr,str) (memset(&MPIDI_Trace_buf[tt].S[(nbr & SEQMASK)],0,sizeof(str))); +#else +int recv_status; +int send_status; +int posted_recv; +#define MPIDI_SET_PR_REC(rreq,buf,ct,dt,pami_id,rank,tag,comm,is_blk) 0 +#define MPIDI_GET_S_REC(dest,sreq,ctx,isSync,dataSize) 0 +#define TRACE_SET_S_BIT(dd,ii,mbr) 0 +#define TRACE_SET_R_BIT(dd,ii,mbr) 0 +#define TRACE_SET_S_VAL(dd,ii,mbr,val) 0 +#define TRACE_SET_R_VAL(dd,ii,mbr,val) 0 +#define TRACE_SET_REQ_VAL(ww,val1) 0 +#define TRACE_MEMSET_R(tt,nbr,str) 0 #endif /* MPIDI_TRACE */ #endif /* include_mpidi_trace_h */ diff --git a/src/mpid/pamid/include/mpidpre.h b/src/mpid/pamid/include/mpidpre.h index ebb99af..29039e2 100644 --- a/src/mpid/pamid/include/mpidpre.h +++ b/src/mpid/pamid/include/mpidpre.h @@ -58,9 +58,6 @@ #include "mpidi_hooks.h" #include "mpidi_thread.h" #include "mpidi_util.h" -#ifdef MPIDI_TRACE -#include "mpidi_trace.h" -#endif #ifdef DYNAMIC_TASKING #define HAVE_GPID_ROUTINES diff --git a/src/mpid/pamid/src/mpid_recvq.c b/src/mpid/pamid/src/mpid_recvq.c index 8a7f43b..3cd9cae 100644 --- a/src/mpid/pamid/src/mpid_recvq.c +++ b/src/mpid/pamid/src/mpid_recvq.c @@ -316,9 +316,7 @@ MPIDI_Recvq_FDU(int source, pami_task_t pami_source, int tag, int context_id, in #endif MPIDI_Recvq_remove(MPIDI_Recvq.unexpected, rreq, prev_rreq); found = TRUE; -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ2=1; -#endif + TRACE_SET_R_BIT((rreq->mpid.partner_id),(rreq->mpid.idx),fl.f.matchedInUQ2); goto fn_exit; } #ifdef OUT_OF_ORDER_HANDLING @@ -507,11 +505,10 @@ MPIDI_Recvq_AEU(MPID_Request *newreq, int source, pami_task_t pami_source, int t MPID_Request *rreq; rreq = newreq; rreq->kind = MPID_REQUEST_RECV; -#ifdef MPIDI_TRACE - rreq->mpid.envelope.msginfo.MPIseqno=-1; - rreq->mpid.envelope.length=0; - rreq->mpid.envelope.data=NULL; -#endif + TRACE_MEMSET_R(pami_source,msg_seqno,recv_status); + TRACE_SET_REQ_VAL(rreq->mpid.envelope.msginfo.MPIseqno,-1); + TRACE_SET_REQ_VAL(rreq->mpid.envelope.length,-1); + TRACE_SET_REQ_VAL(rreq->mpid.envelope.data,(void *) 0); #ifndef OUT_OF_ORDER_HANDLING MPIDI_Request_setMatch(rreq, tag, source, context_id); MPIDI_Recvq_append(MPIDI_Recvq.unexpected, rreq); @@ -519,13 +516,6 @@ MPIDI_Recvq_AEU(MPID_Request *newreq, int source, pami_task_t pami_source, int t MPID_Request *q; MPIDI_In_cntr_t *in_cntr; int insert, i; -#ifdef MPIDI_TRACE - int idx; - idx=(msg_seqno & SEQMASK); - recv_status *rstatus; - rstatus=&MPIDI_In_cntr[pami_source].R[idx]; - memset(rstatus,0,sizeof(recv_status)); -#endif in_cntr = &MPIDI_In_cntr[pami_source]; MPIDI_Request_setMatch(rreq, tag, source, context_id); /* mpi rank needed */ @@ -552,16 +542,14 @@ MPIDI_Recvq_AEU(MPID_Request *newreq, int source, pami_task_t pami_source, int t MPIDI_Recvq_append(MPIDI_Recvq.unexpected, rreq); } } -#ifdef MPIDI_TRACE - rstatus->req=rreq; - rstatus->msgid=msg_seqno; - rstatus->ool=1; - rstatus->rtag=tag; - rstatus->rctx=context_id; - rreq->mpid.idx=idx; - rstatus->rsource=pami_source; - rreq->mpid.partner_id=pami_source; -#endif + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),req,rreq); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),msgid,msg_seqno); + TRACE_SET_R_BIT(pami_source,(msg_seqno & SEQMASK),fl.f.ool); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),rtag,tag); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),rctx,context_id); + TRACE_SET_REQ_VAL(rreq->mpid.idx,(msg_seqno & SEQMASK)); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),rsource,pami_source); + TRACE_SET_REQ_VAL(rreq->mpid.partner_id,pami_source); if (((int)(in_cntr->nMsgs - msg_seqno)) < 0) { /* seqno > nMsgs, out of order */ MPIDI_Recvq_enqueue_ool(pami_source,rreq); diff --git a/src/mpid/pamid/src/mpid_recvq.h b/src/mpid/pamid/src/mpid_recvq.h index ba5a57c..707fb66 100644 --- a/src/mpid/pamid/src/mpid_recvq.h +++ b/src/mpid/pamid/src/mpid_recvq.h @@ -166,11 +166,9 @@ MPIDI_Recvq_FDU_or_AEP(MPID_Request *newreq, int source, pami_task_t pami_source so we need to allocate a new request and add it to the posted queue */ rreq = newreq; -#ifdef MPIDI_TRACE - rreq->mpid.envelope.msginfo.MPIseqno=-1; - rreq->mpid.envelope.length=-1; - rreq->mpid.envelope.data=NULL; -#endif + TRACE_SET_REQ_VAL(rreq->mpid.envelope.msginfo.MPIseqno,-1); + TRACE_SET_REQ_VAL(rreq->mpid.envelope.length,-1); + TRACE_SET_REQ_VAL(rreq->mpid.envelope.data,(void *) 0); rreq->kind = MPID_REQUEST_RECV; MPIDI_Request_setMatch(rreq, tag, source, context_id); MPIDI_Recvq_append(MPIDI_Recvq.posted, rreq); @@ -237,13 +235,7 @@ MPIDI_Recvq_FDP(size_t source, pami_task_t pami_source, int tag, int context_id, #ifdef USE_STATISTICS unsigned search_length = 0; #endif -#ifdef MPIDI_TRACE - int idx; - idx=(msg_seqno & SEQMASK); - recv_status *rstatus; - memset(&MPIDI_Trace_buf[pami_source].R[idx],0,sizeof(recv_status)); - rstatus=&MPIDI_Trace_buf[pami_source].R[idx]; -#endif + TRACE_MEMSET_R(pami_source,msg_seqno,recv_status); rreq = MPIDI_Recvq.posted_head; @@ -282,19 +274,17 @@ MPIDI_Recvq_FDP(size_t source, pami_task_t pami_source, int tag, int context_id, if (flag) #endif { -#ifdef MPIDI_TRACE - rstatus->req=rreq; - rstatus->msgid=msg_seqno; - rstatus->posted=1; - rstatus->bufadd=rreq->mpid.userbuf; - rstatus->len=rreq->mpid.envelope.length; - rstatus->rtag=tag; - rstatus->rctx=context_id; - rstatus->matchedInHH=1; - rstatus->rsource=pami_source; - rreq->mpid.idx=idx; - rreq->mpid.partner_id=pami_source; -#endif + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),req,rreq); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),msgid,msg_seqno); + TRACE_SET_R_BIT(pami_source,(msg_seqno & SEQMASK),fl.f.posted); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),bufadd,rreq->mpid.userbuf); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),len,rreq->mpid.envelope.length); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),rtag,tag); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),rctx,context_id); + TRACE_SET_R_VAL(pami_source,(msg_seqno & SEQMASK),rsource,pami_source); + TRACE_SET_R_BIT(pami_source,(msg_seqno & SEQMASK),fl.f.matchedInHH); + TRACE_SET_REQ_VAL(rreq->mpid.idx,(msg_seqno & SEQMASK)); + TRACE_SET_REQ_VAL(rreq->mpid.partner_id,pami_source); #ifdef OUT_OF_ORDER_HANDLING MPIDI_Request_setPeerRank_pami(rreq, pami_source); #endif diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c index bcdf5d5..8b8c774 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c @@ -202,7 +202,7 @@ MPIDI_RecvCB(pami_context_t context, MPIDI_Request_cpyPeerRequestH(rreq, msginfo); MPIDI_Request_setSync (rreq, msginfo->isSync); MPIDI_Request_setRzv (rreq, 0); - + TRACE_SET_R_VAL(source,(rreq->mpid.PR_idx),len,sndlen); /* --------------------------------------- */ /* We have to fill in the callback info. */ /* --------------------------------------- */ @@ -295,10 +295,8 @@ MPIDI_RecvCB(pami_context_t context, /* -------------------------------------------------- */ recv->addr = rreq->mpid.uebuf; } -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].comp_in_HH=2; - MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; -#endif + TRACE_SET_R_VAL(source,(rreq->mpid.idx),fl.f.comp_in_HH,2); + TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); fn_exit_eager: #ifdef OUT_OF_ORDER_HANDLING diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c index 45a7963..8b00c48 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c @@ -111,12 +111,10 @@ MPIDI_RecvRzvCB_impl(pami_context_t context, rreq->mpid.envelope.data = envelope->data; #endif rreq->mpid.envelope.length = envelope->length; -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].req=rreq; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].rzv=1; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].rlen=envelope->length; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].sync=msginfo->isSync; -#endif + TRACE_SET_R_VAL(source,(rreq->mpid.idx),req,rreq); + TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,envelope->length); + TRACE_SET_R_VAL(source,(rreq->mpid.idx),fl.f.sync,msginfo->isSync); + TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.rzv); if (TOKEN_FLOW_CONTROL_ON) { #if TOKEN_FLOW_CONTROL @@ -147,11 +145,9 @@ MPIDI_RecvRzvCB_impl(pami_context_t context, else { MPIDI_RendezvousTransfer(context, rreq); -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].sync_com_in_HH=1; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].matchedInHH=1; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; -#endif + TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.sync_com_in_HH); + TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.matchedInHH); + TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); } MPID_Request_discard(newreq); } diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c index 14a6f2e..552c636 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_short.c @@ -196,11 +196,9 @@ MPIDI_RecvShortCB(pami_context_t context, if (sndlen > 0) memcpy(rcvbuf, sndbuf, sndlen); -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].rlen=sndlen; - MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].comp_in_HH=1; - MPIDI_Trace_buf[(PAMIX_Endpoint_query(sender))].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; -#endif + TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen); + TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH); + TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); MPIDI_Request_complete(rreq); fn_exit_short: diff --git a/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c b/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c index 3f80fae..931404c 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_callback_util.c @@ -73,18 +73,14 @@ MPIDI_Callback_process_unexp(MPID_Request *newreq, #endif MPID_assert(!sndlen || rreq->mpid.uebuf != NULL); -#ifdef MPIDI_TRACE - int idx=(msginfo->MPIseqno & SEQMASK); - int source=PAMIX_Endpoint_query(sender); - memset(&MPIDI_Trace_buf[source].R[idx],0,sizeof(recv_status)); - MPIDI_Trace_buf[source].R[idx].msgid=msginfo->MPIseqno; - MPIDI_Trace_buf[source].R[idx].rtag=tag; - MPIDI_Trace_buf[source].R[idx].rctx=msginfo->MPIctxt; - MPIDI_Trace_buf[source].R[idx].rlen=sndlen; - MPIDI_Trace_buf[source].R[idx].sync=isSync; - MPIDI_Trace_buf[source].R[idx].rsource=source; - rreq->mpid.idx=idx; -#endif + TRACE_MEMSET_R(PAMIX_Endpoint_query(sender),msg_seqno,recv_status); + TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),msgid,msginfo->MPIseqno); + TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rtag,tag); + TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rctx,msginfo->MPIctxt); + TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rlen,sndlen); + TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),fl.f.sync,isSync); + TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rsource,PAMIX_Endpoint_query(sender)); + TRACE_SET_REQ_VAL(rreq->mpid.idx,(msginfo->MPIseqno & SEQMASK)); if (recv != NULL) { diff --git a/src/mpid/pamid/src/pt2pt/mpidi_control.c b/src/mpid/pamid/src/pt2pt/mpidi_control.c index b39fa51..39629fd 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_control.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_control.c @@ -132,13 +132,9 @@ MPIDI_RecvRzvDoneCB_zerobyte(pami_context_t context, MPIDI_Request_setControl(rreq, original_value); MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS); -#ifdef MPIDI_TRACE - pami_task_t source; - source = MPIDI_Request_getPeerRank_pami(rreq); - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].sync_com_in_HH=1; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].matchedInHH=1; - MPIDI_Trace_buf[source].R[(rreq->mpid.idx)].bufadd=rreq->mpid.userbuf; -#endif + TRACE_SET_R_BIT(MPIDI_Request_getPeerRank_pami(rreq),(rreq->mpid.idx),fl.f.sync_com_in_HH); + TRACE_SET_R_BIT(MPIDI_Request_getPeerRank_pami(rreq),(rreq->mpid.idx),fl.f.matchedInHH); + TRACE_SET_R_VAL(MPIDI_Request_getPeerRank_pami(rreq),(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); MPID_Request_release(rreq); } @@ -228,10 +224,7 @@ MPIDI_RzvAck_proc_req(pami_context_t context, MPID_assert(rc == PAMI_SUCCESS); } #endif -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[(req->mpid.partner_id)].S[(req->mpid.idx)].recvAck=1; -#endif - + TRACE_SET_S_BIT(req->mpid.partner_id,(req->mpid.idx),fl.f.recvAck); MPIDI_SendDoneCB(context, req, PAMI_SUCCESS); } diff --git a/src/mpid/pamid/src/pt2pt/mpidi_done.c b/src/mpid/pamid/src/pt2pt/mpidi_done.c index c71acba..8e68aa9 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_done.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_done.c @@ -33,10 +33,7 @@ MPIDI_SendDoneCB(pami_context_t context, void * clientdata, pami_result_t result) { -#ifdef MPIDI_TRACE - MPID_Request * req = (MPID_Request *) clientdata; - MPIDI_Trace_buf[(req->mpid.partner_id)].S[(req->mpid.idx)].sendComp=1; -#endif + TRACE_SET_S_BIT((((MPID_Request *) clientdata)->mpid.partner_id),(((MPID_Request *) clientdata)->mpid.idx),fl.f.sendComp); MPIDI_SendDoneCB_inline(context, clientdata, result); @@ -198,11 +195,9 @@ void MPIDI_Recvq_process_out_of_order_msgs(pami_task_t src, pami_context_t conte rreq->status.MPI_ERROR = MPI_ERR_TRUNCATE; } -#ifdef MPIDI_TRACE - rreq->mpid.idx = ooreq->mpid.idx; - MPIDI_Trace_buf[src].R[(rreq->mpid.idx)].matchedInOOL=1; - MPIDI_Trace_buf[src].R[(rreq->mpid.idx)].rlen=dt_size; -#endif + TRACE_SET_REQ_VAL(rreq->mpid.idx,ooreq->mpid.idx); + TRACE_SET_R_BIT(src,(rreq->mpid.idx),fl.f.matchedInOOL); + TRACE_SET_R_VAL(src,(rreq->mpid.idx),rlen,dt_size); ooreq->comm = rreq->comm; MPIR_Comm_add_ref(ooreq->comm); ooreq->mpid.userbuf = rreq->mpid.userbuf; diff --git a/src/mpid/pamid/src/pt2pt/mpidi_recv.h b/src/mpid/pamid/src/pt2pt/mpidi_recv.h index 8ff9f10..5bb63df 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_recv.h +++ b/src/mpid/pamid/src/pt2pt/mpidi_recv.h @@ -190,13 +190,7 @@ MPIDI_Recv(void * buf, comm->recvcontext_id + context_offset, &found); #endif -#ifdef MPIDI_TRACE -{ - size_t ll; - ll = count * MPID_Datatype_get_basic_size(datatype); - MPIDI_SET_PR_REC(rreq,buf,count,ll,datatype,pami_source,rank,tag,comm,is_blocking); -} -#endif + MPIDI_SET_PR_REC(rreq,buf,count,datatype,pami_source,rank,tag,comm,is_blocking); /* ----------------------------------------------------------------- */ /* populate request with our data */ @@ -212,6 +206,7 @@ MPIDI_Recv(void * buf, if (unlikely(found)) { + TRACE_SET_R_VAL(pami_source,(rreq->mpid.PR_idx),len,rreq->mpid.uebuflen); MPIDI_RecvMsg_Unexp(rreq, buf, count, datatype); mpi_errno = rreq->status.MPI_ERROR; if (TOKEN_FLOW_CONTROL_ON) { diff --git a/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c index e070efd..71191f9 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c @@ -36,9 +36,7 @@ MPIDI_RecvMsg_Unexp(MPID_Request * rreq, /* The recvnew callback will acknowledge the posted messages */ /* Recv functions will ack the messages that are unexpected */ /* ------------------------------------------------------------ */ -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ=1; -#endif + TRACE_SET_R_BIT((rreq->mpid.partner_id),(rreq->mpid.idx),fl.f.matchedInUQ); if (MPIDI_Request_isRzv(rreq)) { diff --git a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c index 5b00299..805d310 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c @@ -73,15 +73,13 @@ MPIDI_SendMsg_short(pami_context_t context, } #endif MPID_assert(rc == PAMI_SUCCESS); -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].mode=params.dispatch; + TRACE_SET_S_VAL(dest,(sreq->mpid.envelope.msginfo.MPIseqno & SEQMASK),mode,params.dispatch); if (!isSync) { - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].NoComp=1; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendShort=1; - } else - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendEnvelop=1; - -#endif + TRACE_SET_S_BIT(dest,(sreq->mpid.envelope.msginfo.MPIseqno & SEQMASK),fl.f.NoComp); + TRACE_SET_S_BIT(dest,(sreq->mpid.envelope.msginfo.MPIseqno & SEQMASK),fl.f.sendShort); + } else { + TRACE_SET_S_BIT(dest,(sreq->mpid.envelope.msginfo.MPIseqno & SEQMASK),fl.f.sendEnvelop); + } MPIDI_SendDoneCB_inline(context, sreq, PAMI_SUCCESS); #if (MPIDI_STATISTICS) @@ -128,10 +126,8 @@ MPIDI_SendMsg_eager(pami_context_t context, pami_result_t rc; rc = PAMI_Send(context, ¶ms); MPID_assert(rc == PAMI_SUCCESS); -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].mode=MPIDI_Protocols_Eager; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendEager=1; -#endif + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),mode,MPIDI_Protocols_Eager); + TRACE_SET_S_BIT(dest,(sreq->mpid.idx),fl.f.sendEager); } @@ -239,14 +235,12 @@ MPIDI_SendMsg_rzv(pami_context_t context, rc = PAMI_Send_immediate(context, ¶ms); MPID_assert(rc == PAMI_SUCCESS); -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].bufaddr=sreq->mpid.envelope.data; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].mode=MPIDI_Protocols_RVZ; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendRzv=1; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendEnvelop=1; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].memRegion=sreq->mpid.envelope.memregion_used; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].use_pami_get=MPIDI_Process.mp_s_use_pami_get; -#endif + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),bufaddr,sreq->mpid.envelope.data); + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),mode,MPIDI_Protocols_RVZ); + TRACE_SET_S_BIT(dest,(sreq->mpid.idx),fl.f.sendRzv); + TRACE_SET_S_BIT(dest,(sreq->mpid.idx),fl.f.sendEnvelop); + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),fl.f.memRegion,sreq->mpid.envelope.memregion_used); + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),fl.f.use_pami_get,MPIDI_Process.mp_s_use_pami_get); } @@ -299,14 +293,12 @@ MPIDI_SendMsg_rzv_zerobyte(pami_context_t context, rc = PAMI_Send_immediate(context, ¶ms); MPID_assert(rc == PAMI_SUCCESS); -#ifdef MPIDI_TRACE - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].bufaddr=sreq->mpid.envelope.data; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].mode=MPIDI_Protocols_RVZ_zerobyte; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendRzv=1; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].sendEnvelop=1; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].memRegion=sreq->mpid.envelope.memregion_used; - MPIDI_Trace_buf[dest].S[(sreq->mpid.idx)].use_pami_get=MPIDI_Process.mp_s_use_pami_get; -#endif + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),bufaddr,sreq->mpid.envelope.data); + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),mode,MPIDI_Protocols_RVZ_zerobyte); + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),fl.f.memRegion,sreq->mpid.envelope.memregion_used); + TRACE_SET_S_VAL(dest,(sreq->mpid.idx),fl.f.use_pami_get,MPIDI_Process.mp_s_use_pami_get); + TRACE_SET_S_BIT(dest,(sreq->mpid.idx),fl.f.sendRzv); + TRACE_SET_S_BIT(dest,(sreq->mpid.idx),fl.f.sendEnvelop); } @@ -440,10 +432,7 @@ if (!TOKEN_FLOW_CONTROL_ON) { { MPIDI_SendMsg_process_userdefined_dt(sreq, &sndbuf, &data_sz); } -#ifdef MPIDI_TRACE - sreq->mpid.partner_id=dest; - MPIDI_GET_S_REC(sreq,context,isSync,data_sz); -#endif + MPIDI_GET_S_REC(dest_tid,sreq,context,isSync,data_sz); #ifdef OUT_OF_ORDER_HANDLING sreq->mpid.envelope.msginfo.noRDMA=0; @@ -519,10 +508,7 @@ if (!TOKEN_FLOW_CONTROL_ON) { #if TOKEN_FLOW_CONTROL if (!(sreq->mpid.userbufcount)) { -#ifdef MPIDI_TRACE - sreq->mpid.partner_id=dest; - MPIDI_GET_S_REC(sreq,context,isSync,0); -#endif + MPIDI_GET_S_REC(dest_tid,sreq,context,isSync,0); TRACE_ERR("Sending(short,intranode) bytes=%u (short_limit=%u)\n", data_sz, MPIDI_Process.short_limit); MPIU_THREAD_CS_ENTER(MSGQUEUE,0); MPIDI_Piggy_back_tokens(dest,sreq,0); @@ -548,10 +534,7 @@ if (!TOKEN_FLOW_CONTROL_ON) { { MPIDI_SendMsg_process_userdefined_dt(sreq, &sndbuf, &data_sz); } -#ifdef MPIDI_TRACE - sreq->mpid.partner_id=dest; - MPIDI_GET_S_REC(sreq,context,isSync,data_sz); -#endif + MPIDI_GET_S_REC(dest_tid,sreq,context,isSync,data_sz); if (unlikely(PAMIX_Task_is_local(dest_tid) != 0)) noRDMA=1; MPIU_THREAD_CS_ENTER(MSGQUEUE,0); http://git.mpich.org/mpich.git/commitdiff/fe43008c712fe7f33564e7d1179b999708... commit fe43008c712fe7f33564e7d1179b9997087efe36 Author: Su Huang <[email protected]> Date: Tue May 21 14:23:46 2013 -0400 Multi mpi fail with token flow control and euidevelop=deb (ibm) D190610 (ibm) e7d1baff8b4609f5a81f8bd93a52e13d56dc9c30 (ibm) c914bf17412e2d0d16c44d7877ff7bd628599056 Signed-off-by: Michael Blocksome <[email protected]> diff --git a/src/mpid/pamid/src/mpidi_bufmm.c b/src/mpid/pamid/src/mpidi_bufmm.c index 4414359..0134556 100644 --- a/src/mpid/pamid/src/mpidi_bufmm.c +++ b/src/mpid/pamid/src/mpidi_bufmm.c @@ -192,7 +192,7 @@ void MPIDI_calc_tokens(int nTasks,uint *eager_limit_in, unsigned long *buf_mem_i if (MPIDI_tfctrl_enabled < 2) { /* Sometimes we are off by 1 - due to integer arithmetic. */ new_buf_mem_max = (2 * nTasks * MIN_BUF_BKT_SIZE); - if ( new_buf_mem_max <= BUFFER_MEM_MAX ) { + if ( (new_buf_mem_max <= BUFFER_MEM_MAX) && (!application_set_buf_mem) ) { MPIDI_tfctrl_enabled = 2; /* Reset val to mini (64) because the for loop above */ /* would have changed it to 32. */ @@ -206,6 +206,13 @@ void MPIDI_calc_tokens(int nTasks,uint *eager_limit_in, unsigned long *buf_mem_i /* Still not enough ...... Turn off eager send protocol */ MPIDI_tfctrl_enabled = 0; val = 0; + if (((MPIDI_Process.verbose >= MPIDI_VERBOSE_SUMMARY_0)|| (MPIDI_Process.mp_infolevel > 0)) + && (MPIR_Process.comm_world->rank == 0)) { + if ( application_set_buf_mem ) { + printf("ATTENTION: MP_BUFFER_MEM=%d was set too low, eager send protocol is disabled\n", + *buf_mem_in); + } + } } } } diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c index c14c617..ee1c799 100644 --- a/src/mpid/pamid/src/mpidi_env.c +++ b/src/mpid/pamid/src/mpidi_env.c @@ -1213,10 +1213,12 @@ int MPIDI_get_buf_mem(unsigned long *buf_mem,unsigned long *buf_mem_max) int pre_alloc_val=0; unsigned long buf_max_val; int has_error = 0; + extern int application_set_buf_mem; if (cp = getenv("MP_BUFFER_MEM")) { pre_alloc_buf[24] = '\0'; buf_max[24] = '\0'; + application_set_buf_mem=1; if ( (buf_max_cp = strchr(cp, ',')) ) { if ( *(++buf_max_cp) == '\0' ) { /* Error: missing buffer_mem_max */ @@ -1261,7 +1263,7 @@ int MPIDI_get_buf_mem(unsigned long *buf_mem,unsigned long *buf_mem_max) *buf_mem = (int) pre_alloc_val; if (buf_max_val > ONE_SHARED_SEGMENT) *buf_mem = ONE_SHARED_SEGMENT; - if (buf_max_val > *buf_mem_max) + if (buf_max_val != *buf_mem_max) *buf_mem_max = buf_max_val; } else { args_in_error += 1; http://git.mpich.org/mpich.git/commitdiff/d0c1c15acd0ed286c737570399e075f08b... commit d0c1c15acd0ed286c737570399e075f08b63e6fe Author: Haizhu Liu <[email protected]> Date: Wed May 22 10:23:06 2013 -0400 Correct a 0 byte malloc failure on AIX (ibm) D190401 (ibm) 63bc0c36a880f4ce52812aa447ba32d2968aa8f8 Signed-off-by: Michael Blocksome <[email protected]> diff --git a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c index d6308dd..5b00299 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c @@ -355,24 +355,28 @@ MPIDI_SendMsg_process_userdefined_dt(MPID_Request * sreq, { MPID_Segment segment; - sreq->mpid.uebuf = sndbuf = MPIU_Malloc(data_sz); - if (unlikely(sndbuf == NULL)) - { - sreq->status.MPI_ERROR = MPI_ERR_NO_SPACE; - sreq->status.count = 0; - MPID_Abort(NULL, MPI_ERR_NO_SPACE, -1, - "Unable to allocate non-contiguous buffer"); - } - sreq->mpid.uebuf_malloc = mpiuMalloc; - - DLOOP_Offset last = data_sz; - MPID_Segment_init(sreq->mpid.userbuf, - sreq->mpid.userbufcount, - sreq->mpid.datatype, - &segment, - 0); - MPID_Segment_pack(&segment, 0, &last, sndbuf); - MPID_assert(last == data_sz); + if(data_sz != 0) { + sreq->mpid.uebuf = sndbuf = MPIU_Malloc(data_sz); + if (unlikely(sndbuf == NULL)) + { + sreq->status.MPI_ERROR = MPI_ERR_NO_SPACE; + sreq->status.count = 0; + MPID_Abort(NULL, MPI_ERR_NO_SPACE, -1, + "Unable to allocate non-contiguous buffer"); + } + sreq->mpid.uebuf_malloc = mpiuMalloc; + + DLOOP_Offset last = data_sz; + MPID_Segment_init(sreq->mpid.userbuf, + sreq->mpid.userbufcount, + sreq->mpid.datatype, + &segment, + 0); + MPID_Segment_pack(&segment, 0, &last, sndbuf); + MPID_assert(last == data_sz); + } else { + sndbuf = NULL; + } } *_sndbuf = sndbuf; http://git.mpich.org/mpich.git/commitdiff/709a3b3f17abf85d3055824d1a0bd6977e... commit 709a3b3f17abf85d3055824d1a0bd6977e64b2f7 Author: Haizhu Liu <[email protected]> Date: Thu May 2 16:00:06 2013 -0400 Correct dlopen for libpoe on AIX (ibm) 486b8880571b34205490a14e5069fa001552ca0c (ibm) 8b26dda74b1eebf622cac10b3ad004f98a7d9970 (ibm) D189717 Signed-off-by: Michael Blocksome <[email protected]> diff --git a/src/pmi/pmi2/poe/poe2pmi.c b/src/pmi/pmi2/poe/poe2pmi.c index 4fa1fa0..3a3abc0 100644 --- a/src/pmi/pmi2/poe/poe2pmi.c +++ b/src/pmi/pmi2/poe/poe2pmi.c @@ -3,7 +3,6 @@ * (C) 2007 by Argonne National Laboratory. * See COPYRIGHT in top-level directory. */ - #include <dlfcn.h> #include "mpichconf.h" #include "pmi2.h" @@ -85,10 +84,21 @@ int PMI2_Init(int *spawned, int *size, int *rank, int *appnum) int ret; int (*pmi2_init)(int*, int*, int *, int*); - - poeptr = dlopen("libpoe.so",RTLD_NOW|RTLD_GLOBAL); + char *poelibname; + int poeflags; +#ifndef __AIX__ + poelibname = "libpoe.so"; + poeflags = RTLD_NOW|RTLD_GLOBAL; +#else + poeflags = RTLD_NOW|RTLD_GLOBAL|RTLD_MEMBER; + if(sizeof(void *) == 8) + poelibname = "libpoe_r.a(poe64_r.o)"; + else + poelibname = "libpoe_r.a(poe_r.o)"; +#endif + poeptr = dlopen(poelibname, poeflags); if (poeptr == NULL) { - TRACE_ERR("failed to open libpoe.so\n"); + TRACE_ERR("failed to open %s error=%s\n", poelibname, dlerror()); } pmi2_init = (int (*)())dlsym(poeptr, "PMI2_Init"); ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_datatypes.h | 1 + src/mpid/pamid/include/mpidi_trace.h | 136 ++++++++++-- src/mpid/pamid/include/mpidpre.h | 3 - src/mpid/pamid/include/mpix.h | 78 ++++++- src/mpid/pamid/src/mpid_recvq.c | 38 ++--- src/mpid/pamid/src/mpid_recvq.h | 40 ++--- src/mpid/pamid/src/mpidi_bufmm.c | 9 +- src/mpid/pamid/src/mpidi_env.c | 4 +- src/mpid/pamid/src/mpix/mpix.c | 258 +++++++++++++++++++---- src/mpid/pamid/src/pt2pt/mpidi_callback_eager.c | 8 +- src/mpid/pamid/src/pt2pt/mpidi_callback_rzv.c | 18 +- src/mpid/pamid/src/pt2pt/mpidi_callback_short.c | 8 +- src/mpid/pamid/src/pt2pt/mpidi_callback_util.c | 20 +- src/mpid/pamid/src/pt2pt/mpidi_control.c | 15 +- src/mpid/pamid/src/pt2pt/mpidi_done.c | 16 +- src/mpid/pamid/src/pt2pt/mpidi_recv.h | 9 +- src/mpid/pamid/src/pt2pt/mpidi_recvmsg.c | 4 +- src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c | 106 ++++----- src/pmi/pmi2/poe/poe2pmi.c | 18 ++- 19 files changed, 538 insertions(+), 251 deletions(-) hooks/post-receive -- MPICH primary repository