- commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.2-427-g524aeed
by noreply＠mpich.org 02 Sep '16

02 Sep '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 524aeedc0918261477c46b736fa2ce72085d5115 (commit) via 710d18392e4356593ebeea34bf5c1f28e1747645 (commit) via 4f0d6c12a6f20387db09d1e6557ed71fe9882fae (commit) via 532430583eb9ec01d2c20aefafad15874c0da00f (commit) via e049ef4358cdcbab8f3e6ac4da3cefa7cc88ce62 (commit) via 7ededee9220b263f87668b991e8d1c642ff76072 (commit) from 1c63de860856b416864625ef7be4b379e729d99a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/524aeedc0918261477c46b736fa2ce720… commit 524aeedc0918261477c46b736fa2ce72085d5115 Author: Min Si <msi(a)anl.gov> Date: Thu Jun 23 18:01:48 2016 -0500 Added empty symbol for dynamic process routines. In original code, the dynamic process routine can be disabled by setting MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS. Functions exposed to external code must have empty symbols to avoid compiling failure. Signed-off-by: Yanfei Guo <yguo(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 44e05b2..f6959ad 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1016,6 +1016,7 @@ typedef struct MPIDI_Port_Ops { #define MPIDI_PORTFNS_VERSION 1 int MPIDI_CH3_PortFnsInit( MPIDI_PortFns * ); +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS /* Utility routines provided in src/ch3u_port.c for working with connection queues */ int MPIDI_CH3I_Acceptq_enqueue(MPIDI_VC_t * vc, int port_name_tag); @@ -1023,6 +1024,15 @@ int MPIDI_Port_finalize(void); int MPIDI_CH3I_Port_init(int port_name_tag); int MPIDI_CH3I_Port_destroy(int port_name_tag); +#else +/* Need empty symbols to avoid failure at compile time if defined + * MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS. */ +#define MPIDI_CH3I_Acceptq_enqueue(vc, port_name_tag) (MPI_SUCCESS) +#define MPIDI_Port_finalize() (MPI_SUCCESS) + +#define MPIDI_CH3I_Port_init(port_name_tag) (MPI_SUCCESS) +#define MPIDI_CH3I_Port_destroy(port_name_tag) (MPI_SUCCESS) +#endif /*-------------------------- END MPI PORT SECTION --------------------------*/ @@ -1661,8 +1671,14 @@ int MPIDI_CH3_InitCompleted( void ); #endif /* Routines in support of ch3 */ +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS /* Routine to return the tag associated with a port */ int MPIDI_GetTagFromPort( const char *, int * ); +#else +/* Need empty symbol to avoid failure at compile time if defined + * MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS. */ +#define MPIDI_GetTagFromPort(port_name, port_name_tag) (MPI_SUCCESS) +#endif /* Here are the packet handlers */ int MPIDI_CH3_PktHandler_EagerSend( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, http://git.mpich.org/mpich.git/commitdiff/710d18392e4356593ebeea34bf5c1f28e… commit 710d18392e4356593ebeea34bf5c1f28e1747645 Author: Min Si <msi(a)anl.gov> Date: Fri Jun 17 18:08:09 2016 -0500 Added info "timeout" (in seconds) for MPI_Comm_connect. The default threshold of connection timeout is set by MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT. User code can specify per connect threshold through "timeout" info (in seconds). Signed-off-by: Yanfei Guo <yguo(a)anl.gov> diff --git a/src/mpid/ch3/errnames.txt b/src/mpid/ch3/errnames.txt index 3cd2831..9204942 100644 --- a/src/mpid/ch3/errnames.txt +++ b/src/mpid/ch3/errnames.txt @@ -33,6 +33,7 @@ **ch3|pmi_finalize:PMI_Finalize failed **ch3|pmi_finalize %d:PMI_Finalize failed, error %d **ch3|conntimeout:Connection timed out +**ch3|conntimeout %d:Connection timed out in %d seconds **ch3|portclose:Port is unexpectedly closed # diff --git a/src/mpid/ch3/src/ch3u_port.c b/src/mpid/ch3/src/ch3u_port.c index 8dab65d..6fb1ed0 100644 --- a/src/mpid/ch3/src/ch3u_port.c +++ b/src/mpid/ch3/src/ch3u_port.c @@ -252,7 +252,7 @@ static MPIDI_CH3I_Port_connreq_q_t revoked_connreq_q = {NULL, NULL, 0}; #undef FCNAME #define FCNAME MPL_QUOTE(FUNCNAME) static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, - MPIR_Comm **comm_pptr, + int timeout, MPIR_Comm **comm_pptr, MPIDI_VC_t **vc_pptr) { int mpi_errno = MPI_SUCCESS; @@ -290,6 +290,9 @@ static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, MPID_Time_t time_sta, time_now; double time_gap = 0; + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "connect: waiting accept in %d(s)", timeout)); + MPID_Wtime(&time_sta); do { mpi_errno = MPID_Progress_poke(); @@ -300,7 +303,7 @@ static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, MPID_Wtime_diff(&time_sta, &time_now, &time_gap); /* FIXME: not thread-safe */ } while (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_INITED - && (int) time_gap < MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT); + && (int) time_gap < timeout); } switch (connreq->stat) { @@ -317,7 +320,8 @@ static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, MPIDI_CH3I_Port_connreq_q_enqueue(&revoked_connreq_q, connreq); MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, REVOKE); - MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_PORT, "**ch3|conntimeout"); + MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_PORT, "**ch3|conntimeout", + "**ch3|conntimeout %d", timeout); break; case MPIDI_CH3I_PORT_CONNREQ_ERR_CLOSE: @@ -627,10 +631,22 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, if (rank == root) { + int timeout = MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT; + + /* Check if user specifies timeout threshold. */ + if (info != NULL) { + int info_flag = 0; + char info_value[MPI_MAX_INFO_VAL + 1]; + MPIR_Info_get_impl(info, "timeout", MPI_MAX_INFO_VAL, info_value, &info_flag); + if (info_flag) { + timeout = atoi(info_value); + } + } + /* Establish a communicator to communicate with the root on the other side. */ mpi_errno = MPIDI_Create_inter_root_communicator_connect( - port_name, &tmp_comm, &new_vc); + port_name, timeout, &tmp_comm, &new_vc); if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP_LABEL(mpi_errno, no_port); } http://git.mpich.org/mpich.git/commitdiff/4f0d6c12a6f20387db09d1e6557ed71fe… commit 4f0d6c12a6f20387db09d1e6557ed71fe9882fae Author: Min Si <msi(a)anl.gov> Date: Fri Jun 17 17:36:10 2016 -0500 Removed unused function declaration. Signed-off-by: Yanfei Guo <yguo(a)anl.gov> diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index 2202130..44e05b2 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -1019,11 +1019,6 @@ int MPIDI_CH3_PortFnsInit( MPIDI_PortFns * ); /* Utility routines provided in src/ch3u_port.c for working with connection queues */ int MPIDI_CH3I_Acceptq_enqueue(MPIDI_VC_t * vc, int port_name_tag); -#ifdef MPIDI_CH3_CHANNEL_AVOIDS_SELECT -int MPIDI_CH3_Complete_Acceptq_dequeue(MPIDI_VC_t * vc); -#else -#define MPIDI_CH3_Complete_Acceptq_dequeue(vc) MPI_SUCCESS -#endif int MPIDI_Port_finalize(void); int MPIDI_CH3I_Port_init(int port_name_tag); http://git.mpich.org/mpich.git/commitdiff/532430583eb9ec01d2c20aefafad15874… commit 532430583eb9ec01d2c20aefafad15874c0da00f Author: Min Si <msi(a)anl.gov> Date: Fri Jun 17 17:06:35 2016 -0500 Added support for MPI_Comm_connect timeout. To support connection timeout (defined in standard), we add additional handshake protocol in the vc establishing step in accept/connect routines. If no matching accept issued on the server process, the connect call on client process can return after specified threshold MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT. An MPI error MPI_ERR_PORT will be returned. Detailed algorithm is described at the beginning of ch3u_port.c. Signed-off-by: Yanfei Guo <yguo(a)anl.gov> diff --git a/src/mpid/ch3/errnames.txt b/src/mpid/ch3/errnames.txt index a3308bc..3cd2831 100644 --- a/src/mpid/ch3/errnames.txt +++ b/src/mpid/ch3/errnames.txt @@ -32,6 +32,8 @@ **ch3|close_progress:an error occurred while the device was waiting for all open connections to close **ch3|pmi_finalize:PMI_Finalize failed **ch3|pmi_finalize %d:PMI_Finalize failed, error %d +**ch3|conntimeout:Connection timed out +**ch3|portclose:Port is unexpectedly closed # # RMA errors diff --git a/src/mpid/ch3/include/mpid_port.h b/src/mpid/ch3/include/mpid_port.h new file mode 100644 index 0000000..4463d39 --- /dev/null +++ b/src/mpid/ch3/include/mpid_port.h @@ -0,0 +1,200 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * (C) 2016 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ +#ifndef MPID_PORT_H_ +#define MPID_PORT_H_ + +#include "mpl_utlist.h" + +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS + +/* Header file only used by port connect/accept routines (ch3u_port.c) */ + +typedef enum { + MPIDI_CH3I_PORT_CONNREQ_INITED, /* Connection request initialized. */ + MPIDI_CH3I_PORT_CONNREQ_REVOKE, /* Client started revoking a timed out connection request. */ + MPIDI_CH3I_PORT_CONNREQ_ACCEPT, /* Server started accepting a connection request. */ + MPIDI_CH3I_PORT_CONNREQ_ACCEPTED, /* Server successfully accepted this request (i.e., + * client does not revoke it). */ + MPIDI_CH3I_PORT_CONNREQ_ERR_CLOSE, /* Error state -- Server closed unexpectedly. + * (called close_port or finalize while client is + * still waiting acceptance). + * Note: revoke-state request will be directly + * changed to free-state. */ + MPIDI_CH3I_PORT_CONNREQ_FREE, /* Started freeing a connection request. + * (VC has been locally closed and ready for blocking + * wait on termination). + * A connection request can be eventually freed when: + * (1) connection normally completed; + * (2) a timed out connection (revoke) request received + * accept packet; + * (3) server closed unexpectedly.*/ +} MPIDI_CH3I_Port_connreq_stat_t; + +typedef struct MPIDI_CH3I_Port_connreq { + MPIDI_VC_t *vc; + MPIDI_CH3I_Port_connreq_stat_t stat; + struct MPIDI_CH3I_Port_connreq *next; +} MPIDI_CH3I_Port_connreq_t; + +typedef struct MPIDI_CH3I_Port_connreq_q { + MPIDI_CH3I_Port_connreq_t *head; + MPIDI_CH3I_Port_connreq_t *tail; + int size; +} MPIDI_CH3I_Port_connreq_q_t; + +typedef struct MPIDI_CH3I_Port { + int port_name_tag; + MPIDI_CH3I_Port_connreq_q_t accept_connreq_q; + struct MPIDI_CH3I_Port *next; +} MPIDI_CH3I_Port_t; + +typedef struct MPIDI_CH3I_Port_q { + MPIDI_CH3I_Port_t *head; + MPIDI_CH3I_Port_t *tail; + int size; +} MPIDI_CH3I_Port_q_t; + +#define MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, new_stat) \ + (connreq->stat = MPIDI_CH3I_PORT_CONNREQ_##new_stat) + + +/* Start VC closing protocol -- locally close VC. + * It is the 1st step of VC closing protocol (see ch3u_handle_connection.c): + * local_closed / remote_closed -> close_acked -> closed. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_local_close_vc +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_Port_local_close_vc(MPIDI_VC_t * vc) +{ + int mpi_errno = MPI_SUCCESS; + + /* Note that this routine is usually called in comm_release after matched in + * an accept call. Here we do not have comm for the VC, thus just close it + * with a dummy rank (this parameter is only for debugging) */ + if (vc->state == MPIDI_VC_STATE_ACTIVE || vc->state == MPIDI_VC_STATE_REMOTE_CLOSE) { + mpi_errno = MPIDI_CH3U_VC_SendClose(vc, 0 /* dummy rank */); + } + + return mpi_errno; +} + +/*** Utility routines for connection request queues ***/ + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_connreq_q_enqueue +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_Port_connreq_q_enqueue(MPIDI_CH3I_Port_connreq_q_t * connreq_q, + MPIDI_CH3I_Port_connreq_t * connreq) +{ + MPL_LL_APPEND(connreq_q->head, connreq_q->tail, connreq); + connreq_q->size++; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_connreq_q_dequeue +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_Port_connreq_q_dequeue(MPIDI_CH3I_Port_connreq_q_t * connreq_q, + MPIDI_CH3I_Port_connreq_t ** connreq_ptr) +{ + MPIDI_CH3I_Port_connreq_t *connreq = NULL; + + /* delete head */ + if (connreq_q->head != NULL) { + connreq = connreq_q->head; + + MPL_LL_DELETE(connreq_q->head, connreq_q->tail, connreq); + connreq_q->size--; + } + + (*connreq_ptr) = connreq; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_connreq_q_delete +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static inline void MPIDI_CH3I_Port_connreq_q_delete(MPIDI_CH3I_Port_connreq_q_t * connreq_q, + MPIDI_CH3I_Port_connreq_t * connreq) +{ + MPL_LL_DELETE(connreq_q->head, connreq_q->tail, connreq); + connreq_q->size--; +} + + + +/*** Utility routines for issuing ACK packets in accept/connect routine ***/ + +/* Reply ACK packet to server accept routine. Client replies this packet after + * received an accept packet from server. Handled in MPIDI_CH3_PktHandler_AcceptAck. + * ACK - True means client matched with server acceptance; + * ACK - False means client already started revoking, thus acceptance fails. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_issue_accept_ack +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_Port_issue_accept_ack(MPIDI_VC_t * vc, int ack) +{ + int mpi_errno = MPI_SUCCESS; + MPIR_Request *req_ptr = NULL; + MPIDI_CH3_Pkt_t spkt; + MPIDI_CH3_Pkt_accept_ack_t *ack_pkt = &spkt.accept_ack; + + MPIDI_Pkt_init(ack_pkt, MPIDI_CH3_PKT_ACCEPT_ACK); + ack_pkt->ack = ack; + + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "issuing accpet_ack packet to vc %p, ack=%d", vc, ack)); + + mpi_errno = MPIDI_CH3_iStartMsg(vc, ack_pkt, sizeof(MPIDI_CH3_Pkt_t), &req_ptr); + if (mpi_errno != MPI_SUCCESS) + return mpi_errno; + + if (req_ptr != NULL) + MPIR_Request_free(req_ptr); /* Only reduce reference, released after pkt sent. */ + + return mpi_errno; +} + +/* Reply ACK packet to client connect routine. Server replies this packet in + * accept call for dequeued connection request, or replies immediately when + * received a request for non-existing port. Handled in MPIDI_CH3_PktHandler_ConnAck + * on client. + * ACK - True means server started accepting this connection request; + * ACK - False means port does not exist or being closed on server, thus connection + * fails. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_issue_conn_ack +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static inline int MPIDI_CH3I_Port_issue_conn_ack(MPIDI_VC_t * vc, int ack) +{ + int mpi_errno = MPI_SUCCESS; + MPIR_Request *req_ptr = NULL; + MPIDI_CH3_Pkt_t spkt; + MPIDI_CH3_Pkt_conn_ack_t *ack_pkt = &spkt.conn_ack; + + MPIDI_Pkt_init(ack_pkt, MPIDI_CH3_PKT_CONN_ACK); + ack_pkt->ack = ack; + + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "issuing conn_ack packet to vc %p, ack=%d", vc, ack)); + + mpi_errno = MPIDI_CH3_iStartMsg(vc, ack_pkt, sizeof(MPIDI_CH3_Pkt_t), &req_ptr); + if (mpi_errno != MPI_SUCCESS) + return mpi_errno; + + if (req_ptr != NULL) + MPIR_Request_free(req_ptr); /* Only reduce reference, released after pkt sent. */ + + return mpi_errno; +} + +#endif /* MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS */ + +#endif /* MPID_PORT_H_ */ diff --git a/src/mpid/ch3/include/mpidimpl.h b/src/mpid/ch3/include/mpidimpl.h index e7d2456..2202130 100644 --- a/src/mpid/ch3/include/mpidimpl.h +++ b/src/mpid/ch3/include/mpidimpl.h @@ -694,6 +694,10 @@ typedef struct MPIDI_VC /* port name tag */ int port_name_tag; /* added to handle dynamic process mgmt */ +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS + void *connreq_obj; /* pointer to dynamic connection mgmt object */ +#endif + #if defined(MPID_USE_SEQUENCE_NUMBERS) /* Sequence number of the next packet to be sent */ MPID_Seqnum_t seqnum_send; @@ -1015,12 +1019,15 @@ int MPIDI_CH3_PortFnsInit( MPIDI_PortFns * ); /* Utility routines provided in src/ch3u_port.c for working with connection queues */ int MPIDI_CH3I_Acceptq_enqueue(MPIDI_VC_t * vc, int port_name_tag); -int MPIDI_CH3I_Acceptq_dequeue(MPIDI_VC_t ** vc, int port_name_tag); #ifdef MPIDI_CH3_CHANNEL_AVOIDS_SELECT int MPIDI_CH3_Complete_Acceptq_dequeue(MPIDI_VC_t * vc); #else #define MPIDI_CH3_Complete_Acceptq_dequeue(vc) MPI_SUCCESS #endif +int MPIDI_Port_finalize(void); + +int MPIDI_CH3I_Port_init(int port_name_tag); +int MPIDI_CH3I_Port_destroy(int port_name_tag); /*-------------------------- END MPI PORT SECTION --------------------------*/ @@ -1723,6 +1730,15 @@ int MPIDI_CH3_PktHandler_FlowCntlUpdate( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, intptr_t *, MPIR_Request ** ); int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, intptr_t *, MPIR_Request ** ); + +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS +/* packet handlers used in dynamic process connection. */ +int MPIDI_CH3_PktHandler_ConnAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + intptr_t * buflen, MPIR_Request ** rreqp); +int MPIDI_CH3_PktHandler_AcceptAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + intptr_t * buflen, MPIR_Request ** rreqp); +#endif /* end of MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS */ + int MPIDI_CH3_PktHandler_EndCH3( MPIDI_VC_t *, MPIDI_CH3_Pkt_t *, intptr_t *, MPIR_Request ** ); int MPIDI_CH3_PktHandler_Revoke(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, diff --git a/src/mpid/ch3/include/mpidpkt.h b/src/mpid/ch3/include/mpidpkt.h index 1abc133..5566589 100644 --- a/src/mpid/ch3/include/mpidpkt.h +++ b/src/mpid/ch3/include/mpidpkt.h @@ -103,6 +103,11 @@ typedef enum { MPIDI_CH3_PKT_FLOW_CNTL_UPDATE, /* FIXME: Unused */ MPIDI_CH3_PKT_CLOSE, MPIDI_CH3_PKT_REVOKE, +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS + /* Dynamic Connection Management */ + MPIDI_CH3_PKT_CONN_ACK, + MPIDI_CH3_PKT_ACCEPT_ACK, +#endif MPIDI_CH3_PKT_END_CH3, /* The channel can define additional types by defining the value * MPIDI_CH3_PKT_ENUM */ @@ -840,6 +845,16 @@ typedef struct MPIDI_CH3_Pkt_close { int ack; } MPIDI_CH3_Pkt_close_t; +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS +/* packet types used in dynamic process connection. */ +typedef struct MPIDI_CH3_Pkt_conn_ack { + MPIDI_CH3_Pkt_type_t type; + int ack; +} MPIDI_CH3_Pkt_conn_ack_t; + +typedef MPIDI_CH3_Pkt_conn_ack_t MPIDI_CH3_Pkt_accept_ack_t; +#endif /* end of MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS */ + typedef struct MPIDI_CH3_Pkt_revoke { MPIDI_CH3_Pkt_type_t type; MPIR_Context_id_t revoked_comm; @@ -872,6 +887,10 @@ typedef union MPIDI_CH3_Pkt { MPIDI_CH3_Pkt_ack_t ack; MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; MPIDI_CH3_Pkt_close_t close; +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS + MPIDI_CH3_Pkt_conn_ack_t conn_ack; + MPIDI_CH3_Pkt_accept_ack_t accept_ack; +#endif MPIDI_CH3_Pkt_cas_t cas; MPIDI_CH3_Pkt_cas_resp_t cas_resp; MPIDI_CH3_Pkt_fop_t fop; diff --git a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c index 9f0673e..e66c6d6 100644 --- a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c +++ b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c @@ -563,6 +563,13 @@ int MPIDI_CH3_PktHandler_Init( MPIDI_CH3_PktHandler_Fcn *pktArray[], pktArray[MPIDI_CH3_PKT_CLOSE] = MPIDI_CH3_PktHandler_Close; +#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS + /* Dynamic Connection Management */ + pktArray[MPIDI_CH3_PKT_CONN_ACK] = + MPIDI_CH3_PktHandler_ConnAck; + pktArray[MPIDI_CH3_PKT_ACCEPT_ACK] = + MPIDI_CH3_PktHandler_AcceptAck; +#endif /* Provision for flow control */ pktArray[MPIDI_CH3_PKT_FLOW_CNTL_UPDATE] = 0; diff --git a/src/mpid/ch3/src/ch3u_port.c b/src/mpid/ch3/src/ch3u_port.c index a24d350..8dab65d 100644 --- a/src/mpid/ch3/src/ch3u_port.c +++ b/src/mpid/ch3/src/ch3u_port.c @@ -5,6 +5,27 @@ */ #include "mpidi_ch3_impl.h" +#include "mpid_port.h" + +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT + category : CH3 + type : int + default : 180 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_GROUP_EQ + description : >- + The default time out period in seconds for a connection attempt to the + server communicator where the named port exists but no pending accept. + User can change the value for a specified connection through its info + argument. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ /* * This file replaces ch3u_comm_connect.c and ch3u_comm_accept.c . These @@ -50,6 +71,17 @@ static int SetupNewIntercomm( MPIR_Comm *comm_ptr, int remote_comm_size, MPIR_Comm *intercomm ); static int MPIDI_CH3I_Initialize_tmp_comm(MPIR_Comm **comm_pptr, MPIDI_VC_t *vc_ptr, int is_low_group, int context_id_offset); + +static int MPIDI_CH3I_Acceptq_dequeue(MPIDI_CH3I_Port_connreq_t ** connreq_ptr, int port_name_tag); +static int MPIDI_CH3I_Acceptq_cleanup(MPIDI_CH3I_Port_connreq_q_t * accept_connreq_q); + +static int MPIDI_CH3I_Revokeq_cleanup(void); + +static int MPIDI_CH3I_Port_connreq_create(MPIDI_VC_t * vc, + MPIDI_CH3I_Port_connreq_t ** connreq_ptr); +static int MPIDI_CH3I_Port_connreq_free(MPIDI_CH3I_Port_connreq_t * connreq); + + /* ------------------------------------------------------------------------- */ /* * Structure of this file and the connect/accept algorithm: @@ -78,6 +110,98 @@ static int MPIDI_CH3I_Initialize_tmp_comm(MPIR_Comm **comm_pptr, * routine MPIDI_CH3I_Acceptq_dequeue). This routine returns the matched * virtual connection (VC). * + * ----------------------------------------------------------------------------- + * To support connection timeout, we add additional handshake protocol in above + * VC establishing step. The implementation includes init/destroy processing in + * port open/close routine and MPI_Finalize, packet handlers, and modified code + * in following subroutines: + * - MPIDI_Create_inter_root_communicator_connect + * - and MPIDI_Create_inter_root_communicator_accept + * + * 1. Every connection attempt is described as an *request object* on both sides, + * with following state change (see MPIDI_CH3I_Port_connreq_stat_t). + * - Connecting side: INITED -> REVOKE / ACCEPTED / ERR_CLOSE -> FREE + * - Accepting side : INITED -> ACCEPT -> (ACCEPTED) -> FREE + * + * 2. Handshake protocol: + * I. Connecting side: + * a. After VC is created, a connection *request object* is created with + * INITED state, then we pokes progress to wait the state being changed + * in specified timeout period (default CVAR). + * b. If waiting time exceeded threshold, set state to REVOKE. + * c. Progress engine receives a MPIDI_CH3_PKT_CONN_ACK packet indicating + * whether the other side could accept the request. + * - ACK (TRUE): if request state is INITED, then set to ACCEPTED and + * reply a MPIDI_CH3_PKT_ACCEPT_ACK packet with TRUE ACK; otherwise + * such request is already being revoked, then set to FREE and reply + * packet with FALSE ACK. + * - ACK (FALSE): if request state is INITED, then set to ERR_CLOSE; + * otherwise set to FREE (see MPIDI_CH3_PktHandler_ConnAck). + * d. If the process is still waiting, then it checks the changed state: + * if request is ACCEPTED, then continue connection; otherwise report + * MPI_ERR_PORT -- unexpected port closing. + * + * II. Accepting side: + * a. If port exists, the new VC is enqueued into the accept_queue and + * described as a *request object* with INITED state; otherwise, reply a + * MPIDI_CH3_PKT_CONN_ACK packet with FALSE ACK (closed port) + * (see MPIDI_CH3I_Acceptq_dequeue routine). + * b. After VC is dequeued in accept routine, send a MPIDI_CH3_PKT_CONN_ACK + * packet with TRUE ACK to connecting side and change state to ACCEPT. + * Then we poll progress to wait for state change. + * c. Progress engine receives a MPIDI_CH3_PKT_ACCEPT_ACK packet indicating + * whether the other side can match the acceptance (not being revoked). + * - ACK (TRUE): change request state to ACCEPTED + * - ACK (FALSE): set state to FREE + * (see MPIDI_CH3_PktHandler_AcceptAck). + * d. In accept routine, process checks the changed state. If it became + * ACCEPTED, then finish acceptance; otherwise free this request and VC + * and then wait for next coming request. + * + * 3. Resource cleanup. In case of timed out connection, following user code + * shall be considered as correct program (no description in standard yet). + * Thus the resource of first connect must be cleaned up before exit. + * ================================================ + * Server: Client: + * open_port(); connect(); ** timed out ** + * accept(); connect(); + * close_port(); + * ================================================ + * (* The most critical part is closing VC because of closing synchronization + * with the other side (see note in MPIDI_CH3I_Port_local_close_vc and + * ch3u_handle_connection.c), VC object cannot be freed before it became + * INACTIVE state, otherwise segfault !) + * + * Here is the design to ensure resource cleanup. It also covers most + * incorrect user code (e.g., no accept, no close_port), except no finalize. + * I. Connecting side : + * (use revoked_connreq_q) + * a. A REVOKE state request is enqueued to revoked_connreq_q (in 2-I-b). + * b. If request changed state INITED->ERR_CLOSE in 2-I-c, we free such + * request in the connect call before return (step 2-I-d). + * c. If REVOKE->FREE in step 2-I-c, we start VC closing in packet handler, + * and free request and VC at finalize (see MPIDI_CH3I_Revokeq_cleanup). + * d. For any REVOKE state request at finalize, both request object and VC + * will be eventually freed once the other side started finalize. + * + * II. Accepting side : + * (use port_queue, port->accept_queue, global unexpected_queue) + * a. We create a port object for every opened port and each of them holds + * a separate accept queue (to ensure no mess in multiple-ports). + * b. If request becomes FREE in 2-II-a, we start VC closing there and + * enqueue it to global unexpected_queue. + * c. If request updated state ACCEPT->FREE in 2-II-c, we start VC closing + * in packet handler and free both request and VC in accept call (2-II-d). + * d. For any requests still in port->accept_queue at close_port or finalize, + * issue MPIDI_CH3_PKT_CONN_ACK packet with FALSE ACK to the other side, + * then blocking free both VC and request object there. It also ensures + * 3-I-d (see MPIDI_CH3I_Acceptq_cleanup). + * e. For any requests in global unexpected_queue, we blocking free both VC + * and request at finalize (see MPIDI_Port_finalize). + * + * - END of connection timeout support. + * ----------------------------------------------------------------------------- + * * Once both sides have established there VC, they both invoke * MPIDI_CH3I_Initialize_tmp_comm to create a temporary intercommunicator. * A temporary intercommunicator is constructed so that we can use @@ -102,7 +226,21 @@ static int MPIDI_CH3I_Initialize_tmp_comm(MPIR_Comm **comm_pptr, */ /* ------------------------------------------------------------------------- */ -/* + +/*** Queues for supporting connection timeout ***/ + +/* Server side queues + * - unexpected connection requests queue (global) + * - active port queue */ +static MPIDI_CH3I_Port_connreq_q_t unexpt_connreq_q = {NULL, NULL, 0}; +static MPIDI_CH3I_Port_q_t active_portq = {NULL, NULL, 0}; + +/* Client side queue + * - revoked connection requests (i.e., timeout) */ +static MPIDI_CH3I_Port_connreq_q_t revoked_connreq_q = {NULL, NULL, 0}; + + +/* * These next two routines are used to create a virtual connection * (VC) and a temporary intercommunicator that can be used to * communicate between the two "root" processes for the @@ -121,6 +259,8 @@ static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, MPIR_Comm *tmp_comm; MPIDI_VC_t *connect_vc = NULL; int port_name_tag; + MPIDI_CH3I_Port_connreq_t *connreq = NULL; + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CREATE_INTER_ROOT_COMMUNICATOR_CONNECT); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CREATE_INTER_ROOT_COMMUNICATOR_CONNECT); @@ -140,6 +280,67 @@ static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, MPIR_ERR_POP(mpi_errno); } + mpi_errno = MPIDI_CH3I_Port_connreq_create(connect_vc, &connreq); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Can't create communicator connection object."); + + /* Poke progress to wait server response for the connection request + * before timed out. The response is handled in MPIDI_CH3_PktHandler_ConnResp + * in progress.*/ + { + MPID_Time_t time_sta, time_now; + double time_gap = 0; + + MPID_Wtime(&time_sta); + do { + mpi_errno = MPID_Progress_poke(); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); + + MPID_Wtime(&time_now); + MPID_Wtime_diff(&time_sta, &time_now, &time_gap); + /* FIXME: not thread-safe */ + } while (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_INITED + && (int) time_gap < MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT); + } + + switch (connreq->stat) { + case MPIDI_CH3I_PORT_CONNREQ_ACCEPTED: + /* Successfully matched an acceptance, then finish connection. */ + MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT, VERBOSE, "Matched with server accept"); + break; + + case MPIDI_CH3I_PORT_CONNREQ_INITED: + /* Connection timed out. + * Enqueue to revoked queue. Packet handler will notify server when + * when server starts on it. The request will be released at finalize. */ + MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT, VERBOSE, "Connection timed out"); + + MPIDI_CH3I_Port_connreq_q_enqueue(&revoked_connreq_q, connreq); + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, REVOKE); + MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_PORT, "**ch3|conntimeout"); + break; + + case MPIDI_CH3I_PORT_CONNREQ_ERR_CLOSE: + /* Unexpected port closing on server. + * The same as no port case, return MPI_ERR_PORT. Now we caught the error, + * close vc and free connection request at fn_fail. */ + MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT, VERBOSE, + "Error - remote closed without matching this connection"); + + mpi_errno = MPIDI_CH3I_Port_local_close_vc(connreq->vc); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, FREE); + MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_PORT, "**ch3|portclose"); + break; + + default: + /* Unexpected status, internal error. */ + MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_INTERN, "**unknown"); + break; + } + mpi_errno = MPIDI_CH3I_Initialize_tmp_comm(&tmp_comm, connect_vc, 1, port_name_tag); if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP(mpi_errno); @@ -148,10 +349,18 @@ static int MPIDI_Create_inter_root_communicator_connect(const char *port_name, *comm_pptr = tmp_comm; *vc_pptr = connect_vc; + MPL_free(connreq); + fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CREATE_INTER_ROOT_COMMUNICATOR_CONNECT); return mpi_errno; fn_fail: + if (connreq != NULL) { + int mpi_errno2 = MPI_SUCCESS; + mpi_errno2 = MPIDI_CH3I_Port_connreq_free(connreq); + if (mpi_errno2) + MPIR_ERR_ADD(mpi_errno, mpi_errno2); + } goto fn_exit; } @@ -171,6 +380,8 @@ static int MPIDI_Create_inter_root_communicator_accept(const char *port_name, MPIDI_VC_t *new_vc = NULL; MPID_Progress_state progress_state; int port_name_tag; + MPIDI_CH3I_Port_connreq_t *connreq = NULL; + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CREATE_INTER_ROOT_COMMUNICATOR_ACCEPT); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CREATE_INTER_ROOT_COMMUNICATOR_ACCEPT); @@ -183,28 +394,77 @@ static int MPIDI_Create_inter_root_communicator_accept(const char *port_name, /* FIXME: Describe the algorithm used here, and what routine is user on the other side of this connection */ - /* dequeue the accept queue to see if a connection with the - root on the connect side has been formed in the progress - engine (the connection is returned in the form of a vc). If - not, poke the progress engine. */ + /* dequeue the accept queue to see if a connection request with + * the root on the connect side has been formed in the progress + * engine. If not, poke the progress engine; If a new connection + * request has be found, then we start accepting such request by + * sending an ACK packet to client; Pork progress engine unless + * we get response from client (state changed). */ MPID_Progress_start(&progress_state); - for(;;) - { - MPIDI_CH3I_Acceptq_dequeue(&new_vc, port_name_tag); - if (new_vc != NULL) - { - break; - } + for (;;) { + int matched = 0; - mpi_errno = MPID_Progress_wait(&progress_state); - /* --BEGIN ERROR HANDLING-- */ - if (mpi_errno) - { - MPID_Progress_end(&progress_state); - MPIR_ERR_POP(mpi_errno); - } - /* --END ERROR HANDLING-- */ + if (connreq == NULL) { + mpi_errno = MPIDI_CH3I_Acceptq_dequeue(&connreq, port_name_tag); + if (mpi_errno) + MPIR_ERR_POP(mpi_errno); + } + + if (connreq != NULL && connreq->stat == MPIDI_CH3I_PORT_CONNREQ_INITED) { + new_vc = connreq->vc; + + /* locally accept */ + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, ACCEPT); + + /* Send connection ACK: accept to client */ + mpi_errno = MPIDI_CH3I_Port_issue_conn_ack(connreq->vc, TRUE /*accept*/); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Cannot issue acceptance packet"); + + MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT, VERBOSE, + "Sent acceptance to client, waiting for ACK"); + } + + /* Wait either new connection request or response packet for existing one. */ + mpi_errno = MPID_Progress_wait(&progress_state); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPID_Progress_end(&progress_state); + MPIR_ERR_POP(mpi_errno); + } + /* --END ERROR HANDLING-- */ + + /* Packet handler received response from client, check updated state */ + if (connreq != NULL && connreq->stat != MPIDI_CH3I_PORT_CONNREQ_ACCEPT) { + switch (connreq->stat) { + case MPIDI_CH3I_PORT_CONNREQ_ACCEPTED: + /* Matched, now finish acceptance. */ + MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT, VERBOSE, "Matched with client connect"); + matched = 1; /* leave loop */ + break; + + case MPIDI_CH3I_PORT_CONNREQ_FREE: + /* Client revoked, free connection request.*/ + MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT, VERBOSE, + "Connection is already closed on client"); + + /* Client already started vc closing process, thus it is safe to + * blocking wait here till vc freed. */ + mpi_errno = MPIDI_CH3I_Port_connreq_free(connreq); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); + + connreq = NULL; + break; /* continue while loop */ + default: + /* report internal error -- unexpected state */ + MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_INTERN, "**unknown"); + break; + } + } + + if (matched) + break; } MPID_Progress_end(&progress_state); @@ -216,6 +476,8 @@ static int MPIDI_Create_inter_root_communicator_accept(const char *port_name, *comm_pptr = tmp_comm; *vc_pptr = new_vc; + MPL_free(connreq); + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT,VERBOSE,(MPL_DBG_FDEST, "new_vc=%p", new_vc)); @@ -224,6 +486,12 @@ fn_exit: return mpi_errno; fn_fail: + if(connreq != NULL) { + int mpi_errno2 = MPI_SUCCESS; + mpi_errno2 = MPIDI_CH3I_Port_connreq_free(connreq); + if (mpi_errno2) + MPIR_ERR_ADD(mpi_errno, mpi_errno2); + } goto fn_exit; } @@ -537,6 +805,7 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, } no_port: { + int mpi_errno_noport = MPI_SUCCESS; int mpi_errno2 = MPI_SUCCESS; /* broadcast error notification to other processes */ @@ -544,7 +813,11 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, recv_ints[0] = -1; recv_ints[1] = -1; recv_ints[2] = -1; - MPIR_ERR_SET1(mpi_errno, MPI_ERR_PORT, "**portexist", "**portexist %s", port_name); + + /* append no port error message */ + MPIR_ERR_SET1(mpi_errno_noport, MPI_ERR_PORT, "**portexist", "**portexist %s", port_name); + MPIR_ERR_ADD(mpi_errno_noport, mpi_errno); + mpi_errno = mpi_errno_noport; /* notify other processes to return an error */ MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"broadcasting 3 ints: error case"); @@ -1250,18 +1523,11 @@ static int FreeNewVC( MPIDI_VC_t *new_vc ) designed and documented. */ -typedef struct MPIDI_CH3I_Acceptq_s -{ - struct MPIDI_VC *vc; - int port_name_tag; - struct MPIDI_CH3I_Acceptq_s *next; -} -MPIDI_CH3I_Acceptq_t; - -static MPIDI_CH3I_Acceptq_t * acceptq_head=0; -static int maxAcceptQueueSize = 0; -static int AcceptQueueSize = 0; - +/* Enqueue a connection request from client. This routine is called from netmod + * (i.e., TCP) if received dynamic connection from others. If port exists, we + * enqueue the request to that port's accept queue to wait for an accept call to + * serve it; otherwise, such request should be discarded, thus we immediately send + * nack back to client and start closing. */ #undef FUNCNAME #define FUNCNAME MPIDI_CH3I_Acceptq_enqueue #undef FCNAME @@ -1269,91 +1535,501 @@ static int AcceptQueueSize = 0; int MPIDI_CH3I_Acceptq_enqueue(MPIDI_VC_t * vc, int port_name_tag ) { int mpi_errno=MPI_SUCCESS; - MPIDI_CH3I_Acceptq_t *q_item; - MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_ACCEPTQ_ENQUEUE); + MPIDI_CH3I_Port_connreq_t *connreq = NULL; + MPIDI_CH3I_Port_t *port = NULL; + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_ACCEPTQ_ENQUEUE); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_ACCEPTQ_ENQUEUE); - /* FIXME: Use CHKPMEM */ - q_item = (MPIDI_CH3I_Acceptq_t *) - MPL_malloc(sizeof(MPIDI_CH3I_Acceptq_t)); - /* --BEGIN ERROR HANDLING-- */ - if (q_item == NULL) - { - mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDI_CH3I_Acceptq_t" ); - goto fn_exit; - } - /* --END ERROR HANDLING-- */ + MPL_LL_SEARCH_SCALAR(active_portq.head, port, port_name_tag, port_name_tag); + + /* Find port object by using port_name_tag. */ + mpi_errno = MPIDI_CH3I_Port_connreq_create(vc, &connreq); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Can't create communicator connection object."); - q_item->vc = vc; - q_item->port_name_tag = port_name_tag; + /* No port exists if port is not opened or already closed (incorrect user code). + * Thus we just start closing VC here. */ + if (port == NULL) { + /* Notify connecting client. */ + mpi_errno = MPIDI_CH3I_Port_issue_conn_ack(connreq->vc, FALSE /* closed port */); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); - /* Keep some statistics on the accept queue */ - AcceptQueueSize++; - if (AcceptQueueSize > maxAcceptQueueSize) - maxAcceptQueueSize = AcceptQueueSize; + /* Start VC closing protocol. */ + mpi_errno = MPIDI_CH3I_Port_local_close_vc(connreq->vc); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); - /* FIXME: Stack or queue? */ - MPL_DBG_MSG_P(MPIDI_CH3_DBG_CONNECT,TYPICAL,"vc=%p:Enqueuing accept connection",vc); - q_item->next = acceptq_head; - acceptq_head = q_item; - - fn_exit: + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, FREE); + + /* Enqueue unexpected VC to avoid waiting progress recursively. + * these VCs will be freed in finalize. */ + MPIDI_CH3I_Port_connreq_q_enqueue(&unexpt_connreq_q, connreq); + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "Enqueued conn %p to unexpected queue with tag %d, vc=%p", + connreq, port_name_tag, vc)); + } + else { + /* Enqueue to accept queue, thus next accept call can serve it. */ + MPIDI_CH3I_Port_connreq_q_enqueue(&port->accept_connreq_q, connreq); + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "Enqueued conn %p to accept queue with tag %d, vc=%p", + connreq, port_name_tag, vc)); + + /* signal for new enqueued VC, thus progress wait can return in accept. */ + MPIDI_CH3_Progress_signal_completion(); + } + + fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_ACCEPTQ_ENQUEUE); return mpi_errno; + fn_fail: + if (connreq) + MPIDI_CH3I_Port_connreq_free(connreq); + goto fn_exit; } -/* Attempt to dequeue a vc from the accept queue. If the queue is - empty or the port_name_tag doesn't match, return a NULL vc. */ +/* Attempt to dequeue a connection request from the accept queue. If the queue + * is empty return a NULL object. */ #undef FUNCNAME #define FUNCNAME MPIDI_CH3I_Acceptq_dequeue #undef FCNAME #define FCNAME MPL_QUOTE(FUNCNAME) -int MPIDI_CH3I_Acceptq_dequeue(MPIDI_VC_t ** vc, int port_name_tag) +int MPIDI_CH3I_Acceptq_dequeue(MPIDI_CH3I_Port_connreq_t ** connreq_ptr, int port_name_tag) { int mpi_errno=MPI_SUCCESS; - MPIDI_CH3I_Acceptq_t *q_item, *prev; - MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_ACCEPTQ_DEQUEUE); + MPIDI_CH3I_Port_t *port = NULL; + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_ACCEPTQ_DEQUEUE); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_ACCEPTQ_DEQUEUE); - *vc = NULL; - q_item = acceptq_head; - prev = q_item; + /* Find port object by using port_name_tag. */ + MPL_LL_SEARCH_SCALAR(active_portq.head, port, port_name_tag, port_name_tag); + MPIR_Assert(port != NULL); /* Port is always initialized in open_port. */ - while (q_item != NULL) - { - if (q_item->port_name_tag == port_name_tag) - { - *vc = q_item->vc; + MPIDI_CH3I_Port_connreq_q_dequeue(&port->accept_connreq_q, connreq_ptr); + if ((*connreq_ptr) != NULL) { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "conn=%p:Dequeued accept connection with tag %d, vc=%p", + (*connreq_ptr), port_name_tag, (*connreq_ptr)->vc)); + } - if ( q_item == acceptq_head ) - acceptq_head = q_item->next; - else - prev->next = q_item->next; + MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_ACCEPTQ_DEQUEUE); + return mpi_errno; +} - MPL_free(q_item); - AcceptQueueSize--; - break;; - } - else - { - prev = q_item; - q_item = q_item->next; - } +/* Clean up received new VCs that are not accepted or closed in accept + * calls (e.g., mismatching accept and connect).This routine is called in + * MPIDI_CH3I_Port_destroy(close_port) and MPIDI_Port_finalize (finalize). + * Note that we already deleted port from active_port queue before cleaning up + * its accept queue, thus no new VC can be enqueued concurrently. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Acceptq_cleanup +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static int MPIDI_CH3I_Acceptq_cleanup(MPIDI_CH3I_Port_connreq_q_t * accept_connreq_q) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3I_Port_connreq_t *connreq = NULL, *connreq_tmp = NULL; + + MPL_LL_FOREACH_SAFE(accept_connreq_q->head, connreq, connreq_tmp) { + MPIDI_CH3I_Port_connreq_q_delete(accept_connreq_q, connreq); + + /* Notify connecting client. */ + mpi_errno = MPIDI_CH3I_Port_issue_conn_ack(connreq->vc, FALSE /* closed port */); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); + + /* Start VC closing protocol. */ + mpi_errno = MPIDI_CH3I_Port_local_close_vc(connreq->vc); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, FREE); + + /* Free connection request (blocking wait till VC closed). */ + mpi_errno = MPIDI_CH3I_Port_connreq_free(connreq); + if (mpi_errno != MPI_SUCCESS) + MPIR_ERR_POP(mpi_errno); } - - mpi_errno = MPIDI_CH3_Complete_Acceptq_dequeue(*vc); - MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT,TYPICAL, - (MPL_DBG_FDEST,"vc=%p:Dequeuing accept connection with tag %d", - *vc,port_name_tag)); + MPIR_Assert(accept_connreq_q->size == 0); - MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_ACCEPTQ_DEQUEUE); + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + + +/*** Utility routines for revoked connection requests ***/ + +/* Clean up revoked requests in connect (e.g., timed out connect). + * We do not want to wait for these VCs being freed in timed out connect, + * because it is blocked till the server calls a matching accept or close_port. + * This routine is called in finalize on client process. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Revokeq_cleanup +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static int MPIDI_CH3I_Revokeq_cleanup(void) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3I_Port_connreq_t *connreq = NULL, *connreq_tmp = NULL; + + MPL_LL_FOREACH_SAFE(revoked_connreq_q.head, connreq, connreq_tmp) { + MPID_Progress_state progress_state; + MPIDI_CH3I_Port_connreq_q_delete(&revoked_connreq_q, connreq); + + /* Blocking wait till the request is freed on server. */ + if (connreq->stat != MPIDI_CH3I_PORT_CONNREQ_FREE) { + MPID_Progress_start(&progress_state); + do { + mpi_errno = MPID_Progress_wait(&progress_state); + /* --BEGIN ERROR HANDLING-- */ + if (mpi_errno != MPI_SUCCESS) { + MPID_Progress_end(&progress_state); + MPIR_ERR_POP(mpi_errno); + } + /* --END ERROR HANDLING-- */ + } while (connreq->stat != MPIDI_CH3I_PORT_CONNREQ_FREE); + MPID_Progress_end(&progress_state); + } + + /* Release connection (blocking wait till VC closed). */ + MPIDI_CH3I_Port_connreq_free(connreq); + } + + MPIR_Assert(revoked_connreq_q.size == 0); + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +/*** Packet handlers exposed to progress engine ***/ + +/* Packet handler to handle response (connection ACK) on client process. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_ConnAck +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_ConnAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + intptr_t * buflen, MPIR_Request ** rreqp) +{ + MPIDI_CH3_Pkt_conn_ack_t *ack_pkt = &pkt->conn_ack; + MPIDI_CH3I_Port_connreq_t *connreq = (MPIDI_CH3I_Port_connreq_t *) (vc->connreq_obj); + int mpi_errno = MPI_SUCCESS; + + MPIR_Assert(connreq != NULL); + + /* Report unknown error, unexpectedly get response for remote + * revoked connection. */ + if (connreq->stat != MPIDI_CH3I_PORT_CONNREQ_INITED && + connreq->stat != MPIDI_CH3I_PORT_CONNREQ_REVOKE) + MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_INTERN, "**unknown"); + + if (ack_pkt->ack == TRUE) { + if (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_INITED) { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "received ACK true for vc %p: inited->accepted", vc)); + + /* Reply to server */ + mpi_errno = MPIDI_CH3I_Port_issue_accept_ack(connreq->vc, TRUE /* accept matched */); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Cannot issue accept-matched packet"); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, ACCEPTED); + } + else if (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_REVOKE) { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "received ACK true for vc %p: revoke->free", vc)); + + /* Reply to server */ + mpi_errno = MPIDI_CH3I_Port_issue_accept_ack(connreq->vc, FALSE /* locally revoked */); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Cannot issue revoke packet"); + + /* Start freeing connection request. + * Note that we do not blocking close VC here, instead, we close VC + * in MPIDI_CH3I_Revokeq_cleanup at finalize. This is because + * VC close packets might be received following this packet, thus if + * we blocked here we can never read that packet. */ + mpi_errno = MPIDI_CH3I_Port_local_close_vc(connreq->vc); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Cannot locally close VC"); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, FREE); + } + } + else { /* ack == FALSE */ + if (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_INITED) { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "received ACK false for vc %p: inited->err_close", vc)); + + /* Server closed port without issuing accept, client + * connect call will catch this error and return MPI_ERR_PORT. */ + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, ERR_CLOSE); + } + else if (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_REVOKE) { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "received ACK false for vc %p: revoke->free", vc)); + + /* Start VC closing, and set connection ready-for-free. */ + mpi_errno = MPIDI_CH3I_Port_local_close_vc(connreq->vc); + MPIR_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Cannot locally close VC"); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, FREE); + } + } + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +/* Packet handler to handle response (acceptance ACK) on server process. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3_PktHandler_AcceptAck +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +int MPIDI_CH3_PktHandler_AcceptAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + intptr_t * buflen, MPIR_Request ** rreqp) +{ + MPIDI_CH3_Pkt_accept_ack_t *ack_pkt = &pkt->accept_ack; + MPIDI_CH3I_Port_connreq_t *connreq = (MPIDI_CH3I_Port_connreq_t *) (vc->connreq_obj); + int mpi_errno = MPI_SUCCESS; + + MPIR_Assert(connreq != NULL); + + /* Acceptance matched, finish accept. */ + if (ack_pkt->ack == TRUE) { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "received (accept) ACK true for vc %p: accept->match", vc)); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, ACCEPTED); + } + else { + MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CONNECT, VERBOSE, + (MPL_DBG_FDEST, "received (accept) ACK false for vc %p: accept->close", + vc)); + + /* Client already left, close VC. + * Note that accept call does not return when client timed out, + * thus we only change the state and let accept call handle closing. */ + mpi_errno = MPIDI_CH3I_Port_local_close_vc(connreq->vc); + if (mpi_errno) + MPIR_ERR_POP(mpi_errno); + + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, FREE); + } + + *buflen = sizeof(MPIDI_CH3_Pkt_t); + *rreqp = NULL; + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +/*** Routines for connection request creation and freeing ***/ + +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_connreq_create +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static int MPIDI_CH3I_Port_connreq_create(MPIDI_VC_t * vc, MPIDI_CH3I_Port_connreq_t ** connreq_ptr) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3I_Port_connreq_t *connreq = NULL; + + MPIR_CHKPMEM_DECL(1); + MPIR_CHKPMEM_MALLOC(connreq, MPIDI_CH3I_Port_connreq_t *, sizeof(MPIDI_CH3I_Port_connreq_t), + mpi_errno, "comm_conn"); + + connreq->vc = vc; + MPIDI_CH3I_PORT_CONNREQ_SET_STAT(connreq, INITED); + + /* Netmod may not change VC to active when connection established (i.e., sock). + * Instead, it is changed in CH3 layer (e.g., isend, RMA).*/ + if (vc->state == MPIDI_VC_STATE_INACTIVE) + MPIDI_CHANGE_VC_STATE(vc, ACTIVE); + + vc->connreq_obj = (void *) connreq; /* to get connection request in packet handlers */ + *connreq_ptr = connreq; + + fn_exit: + MPIR_CHKPMEM_COMMIT(); return mpi_errno; + fn_fail: + MPIR_CHKPMEM_REAP(); + goto fn_exit; } +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_connreq_free +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +static int MPIDI_CH3I_Port_connreq_free(MPIDI_CH3I_Port_connreq_t * connreq) +{ + int mpi_errno = MPI_SUCCESS; + + /* Skip if connection request is still in revoked state. + * Because packet handler may be talking to server. */ + if (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_REVOKE) + return mpi_errno; + + /* Expected free, the remote side should also start closing too, + * thus we blocking close VC here. */ + if (connreq->stat == MPIDI_CH3I_PORT_CONNREQ_FREE) { + mpi_errno = FreeNewVC(connreq->vc); + } + else { + /* Unexpected free, the remote side might not be able to close + * VC at this point. Thus we cannot blocking close VC. */ + mpi_errno = MPIDI_CH3_VC_Destroy(connreq->vc); + } + + /* Always free connection request. + * Because it is only used in connect/accept routine. */ + MPL_free(connreq); + + return mpi_errno; +} + + +/*** Routines to initialize / destroy dynamic connection ***/ + +/* Initialize port's accept queue. It is called in MPIDI_Open_port. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_init +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +int MPIDI_CH3I_Port_init(int port_name_tag) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3I_Port_t *port = NULL; + + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_PORT_INIT); + MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_PORT_INIT); + + MPIR_CHKPMEM_DECL(1); + MPIR_CHKPMEM_MALLOC(port, MPIDI_CH3I_Port_t *, sizeof(MPIDI_CH3I_Port_t), + mpi_errno, "comm_port"); + + port->port_name_tag = port_name_tag; + port->accept_connreq_q.head = port->accept_connreq_q.tail = 0; + port->accept_connreq_q.size = 0; + port->next = NULL; + + MPL_LL_APPEND(active_portq.head, active_portq.tail, port); + active_portq.size++; + + fn_exit: + MPIR_CHKPMEM_COMMIT(); + MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_PORT_INIT); + return mpi_errno; + fn_fail: + MPIR_CHKPMEM_REAP(); + goto fn_exit; +} + +/* Destroy port's accept queue. It is called in MPIDI_Close_port. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_CH3I_Port_destroy +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +int MPIDI_CH3I_Port_destroy(int port_name_tag) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_CH3I_Port_t *port = NULL; + + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_PORT_DESTROY); + MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_PORT_DESTROY); + + MPL_LL_SEARCH_SCALAR(active_portq.head, port, port_name_tag, port_name_tag); + if (port != NULL) { + MPL_LL_DELETE(active_portq.head, active_portq.tail, port); + + mpi_errno = MPIDI_CH3I_Acceptq_cleanup(&port->accept_connreq_q); + if (mpi_errno) + MPIR_ERR_POP(mpi_errno); + + MPL_free(port); + active_portq.size--; + } + + fn_exit: + MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_PORT_DESTROY); + return mpi_errno; + fn_fail: + goto fn_exit; +} + +/* This routine is called by MPID_Finalize to clean up dynamic connections. */ +#undef FUNCNAME +#define FUNCNAME MPIDI_Port_finalize +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +int MPIDI_Port_finalize(void) +{ + int mpi_errno = MPI_SUCCESS; + + MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_PORT_FINALIZE); + MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_PORT_FINALIZE); + + /* Server side clean up. */ + + /* - Clean up all active ports. + * Note that if a process is both server and client, we will never + * deadlock here because only server cleanup issues closing packets. + * All closing process on client is handled in progress. */ + { + MPIDI_CH3I_Port_t *port = NULL, *port_tmp = NULL; + + MPL_LL_FOREACH_SAFE(active_portq.head, port, port_tmp) { + /* destroy all opening ports. */ + MPL_LL_DELETE(active_portq.head, active_portq.tail, port); + + mpi_errno = MPIDI_CH3I_Acceptq_cleanup(&port->accept_connreq_q); + MPL_free(port); + active_portq.size--; + } + MPIR_Assert(active_portq.size == 0); + } + + /* - Destroy all unexpected connection requests. + * The closing protocol already started when we got them in acceptq_enqueue, + * so just blocking wait for final release. */ + { + MPIDI_CH3I_Port_connreq_t *connreq = NULL, *connreq_tmp = NULL; + + MPL_LL_FOREACH_SAFE(unexpt_connreq_q.head, connreq, connreq_tmp) { + MPIDI_CH3I_Port_connreq_q_delete(&unexpt_connreq_q, connreq); + mpi_errno = MPIDI_CH3I_Port_connreq_free(connreq); + if (mpi_errno) + MPIR_ERR_POP(mpi_errno); + } + MPIR_Assert(unexpt_connreq_q.size == 0); + } + + /* Client side clean up. */ + + /* - Destroy all revoked connection requests. */ + mpi_errno = MPIDI_CH3I_Revokeq_cleanup(); + if (mpi_errno) + MPIR_ERR_POP(mpi_errno); + + fn_exit: + MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_PORT_FINALIZE); + return mpi_errno; + fn_fail: + goto fn_exit; +} #else /* MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS is defined */ #endif /* MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS */ diff --git a/src/mpid/ch3/src/mpid_finalize.c b/src/mpid/ch3/src/mpid_finalize.c index 0bf8586..a5fe615 100644 --- a/src/mpid/ch3/src/mpid_finalize.c +++ b/src/mpid/ch3/src/mpid_finalize.c @@ -95,6 +95,10 @@ int MPID_Finalize(void) /* Re-enabling the close step because many tests are failing * without it, particularly under gforker */ + + mpi_errno = MPIDI_Port_finalize(); + if (mpi_errno) { MPIR_ERR_POP(mpi_errno); } + #if 1 /* FIXME: The close actions should use the same code as the other connection close code */ diff --git a/src/mpid/ch3/src/mpid_port.c b/src/mpid/ch3/src/mpid_port.c index e306338..7cf4fe1 100644 --- a/src/mpid/ch3/src/mpid_port.c +++ b/src/mpid/ch3/src/mpid_port.c @@ -312,6 +312,8 @@ static int MPIDI_Open_port(MPIR_Info *info_ptr, char *port_name) mpi_errno = MPIDI_CH3_Get_business_card(myRank, port_name, len); MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER, VERBOSE, (MPL_DBG_FDEST, "port_name = %s", port_name)); + mpi_errno = MPIDI_CH3I_Port_init(port_name_tag); + fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_OPEN_PORT); return mpi_errno; @@ -339,6 +341,8 @@ static int MPIDI_Close_port(const char *port_name) free_port_name_tag(port_name_tag); + mpi_errno = MPIDI_CH3I_Port_destroy(port_name_tag); + fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CLOSE_PORT); return mpi_errno; http://git.mpich.org/mpich.git/commitdiff/e049ef4358cdcbab8f3e6ac4da3cefa7c… commit e049ef4358cdcbab8f3e6ac4da3cefa7cc88ce62 Author: Min Si <msi(a)anl.gov> Date: Fri Jun 17 17:07:46 2016 -0500 Disable vc state change in tcp if vc started closing. Tcp netmod updated vc to ACTIVE state when connection is established. It can happen in the first message. However, if the first message is vc closing packet, that vc can never be closed. Because, during vc closing, process needs do handshake with remote peer and change state: LOCAL_CLOSE/REMOTE_CLOSE -> CLOSE_ACKED -> CLOSED -> INACTIVE The next state change depends on the current state. If such state is changed LOCAL_CLOSE->ACTIVE in netmod, the closing process can never finish. This patch fixed it. Tcp netmod only updates state when the vc is INACTIVE. Signed-off-by: Yanfei Guo <yguo(a)anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c index 2277dc8..c44b213 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c +++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c @@ -800,7 +800,11 @@ int MPID_nem_tcp_connect(struct MPIDI_VC *const vc) /* We have an active connection, start polling more often */ MPID_nem_tcp_skip_polls = MAX_SKIP_POLLS_ACTIVE; - MPIDI_CHANGE_VC_STATE(vc, ACTIVE); + /* only update VC state when it is not being closed. + * Note that we still need change state here if the VC is passively + * connected (i.e., server in dynamic process connection) */ + if (vc->state == MPIDI_VC_STATE_INACTIVE) + MPIDI_CHANGE_VC_STATE(vc, ACTIVE); if (vc_tcp->state == MPID_NEM_TCP_VC_STATE_DISCONNECTED) { struct sockaddr_in *sock_addr; diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c index 135b69e..fc73e8d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_send.c @@ -213,7 +213,11 @@ int MPID_nem_tcp_conn_est (MPIDI_VC_t *vc) MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_TCP_CONN_EST); - MPIDI_CHANGE_VC_STATE(vc, ACTIVE); + /* only update VC state when it is not being closed. + * Note that we still need change state here if the VC is passively + * connected (i.e., server in dynamic process connection) */ + if (vc->state == MPIDI_VC_STATE_INACTIVE) + MPIDI_CHANGE_VC_STATE(vc, ACTIVE); if (!MPIDI_CH3I_Sendq_empty(vc_tcp->send_queue)) { http://git.mpich.org/mpich.git/commitdiff/7ededee9220b263f87668b991e8d1c642… commit 7ededee9220b263f87668b991e8d1c642ff76072 Author: Min Si <msi(a)il.is.s.u-tokyo.ac.jp> Date: Mon May 23 18:31:58 2016 -0500 Added tests to check comm_connect timeout. test/mpi/errors/spawn/connect_timeout.c checks the timeout implementation in MPI_Comm_connect. Two tests are generated: (1) no accept: no accept call on server; (2) mismatch: number of accept calls is less than amount of connect. Signed-off-by: Yanfei Guo <yguo(a)anl.gov> diff --git a/test/mpi/errors/spawn/Makefile.am b/test/mpi/errors/spawn/Makefile.am index c2f956f..a9f1b42 100644 --- a/test/mpi/errors/spawn/Makefile.am +++ b/test/mpi/errors/spawn/Makefile.am @@ -12,5 +12,12 @@ EXTRA_DIST = testlist ## for all programs that are just built from the single corresponding source ## file, we don't need per-target _SOURCES rules, automake will infer them ## correctly -noinst_PROGRAMS = badport unpub lookup_name +noinst_PROGRAMS = badport unpub lookup_name \ + connect_timeout_no_accept \ + connect_timeout_mismatch +connect_timeout_no_accept_SOURCES = connect_timeout.c +connect_timeout_mismatch_SOURCES = connect_timeout.c + +connect_timeout_no_accept_CPPFLAGS = -DTEST_NOACCEPT $(AM_CPPFLAGS) +connect_timeout_mismatch_CPPFLAGS = -DTEST_MISMATCH $(AM_CPPFLAGS) \ No newline at end of file diff --git a/test/mpi/errors/spawn/connect_timeout.c b/test/mpi/errors/spawn/connect_timeout.c new file mode 100644 index 0000000..2bef859 --- /dev/null +++ b/test/mpi/errors/spawn/connect_timeout.c @@ -0,0 +1,225 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2016 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ +#include "mpi.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "mpitest.h" + +#define IF_VERBOSE(cond, a) if (verbose && (cond)) { printf a ; fflush(stdout); } +#define SERVER_GID 0 +#define CLIENT_GID 1 +#define TIMEOUT "5" + +/* This test checks whether a MPI_COMM_CONNECT can eventually time out and return + * an error of class MPI_ERR_PORT when the named port exists but no pending + * MPI_COMM_ACCEPT on server side. We divide all processes into a server group + * and a client group, the client group issues connect to the server group. + * + * Two test cases are performed in separately: + * (1) No_accept: Server only calls open/close_port while client tries to connect. + * Each connect call should return MPI_ERR_PORT. + * (2) Mismatched: Server only issues one accept while client tries twice connect. + * At least one of the connect call should return MPI_ERR_PORT. */ + +int rank, nproc; +int verbose = 0; + +int test_mismatched_accept(MPI_Comm intra_comm, int gid); +int test_no_accept(MPI_Comm intra_comm, int gid); + +#if defined(TEST_MISMATCH) +#define RUN_TEST(comm, gid) test_mismatched_accept(comm, gid) +#else /* no accept by default */ +#define RUN_TEST(comm, gid) test_no_accept(comm, gid) +#endif + +/* Check whether error clase is equal to the expected one. */ +static inline int check_errno(int mpi_errno, int expected_errno) +{ + int errclass = MPI_SUCCESS; + int err = 0; + + MPI_Error_class(mpi_errno, &errclass); + if (errclass != expected_errno) { + char errstring[MPI_MAX_ERROR_STRING]; + char exp_errstring[MPI_MAX_ERROR_STRING]; + int errstrlen = 0; + + err++; + MPI_Error_string(mpi_errno, errstring, &errstrlen); + MPI_Error_string(expected_errno, exp_errstring, &errstrlen); + fprintf(stderr, "Error: returned wrong error class %s, expected %s\n", + errstring, exp_errstring); + } + + return err; +} + +/* The first process in server group opens port and broadcast to all other + * processes in the world. */ +static inline void open_and_bcast_port(int intra_rank, int gid, char (*port)[]) +{ + int server_rank = 0, local_server_rank = 0; + + /* server root opens port */ + if (intra_rank == 0 && gid == SERVER_GID) { + local_server_rank = rank; + + MPI_Open_port(MPI_INFO_NULL, (*port)); + IF_VERBOSE(1, ("server root: opened port1: <%s>\n", (*port))); + } + + /* broadcast world rank of server root, then broadcast port */ + MPI_Allreduce(&local_server_rank, &server_rank, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Bcast((*port), MPI_MAX_PORT_NAME, MPI_CHAR, server_rank, MPI_COMM_WORLD); +} + +int test_mismatched_accept(MPI_Comm intra_comm, int gid) +{ + char port[MPI_MAX_PORT_NAME]; + MPI_Comm comm = MPI_COMM_NULL; + int mpi_errno = MPI_SUCCESS; + int errs = 0; + int intra_rank, intra_nproc; + int server_rank = 0, local_server_rank = 0; + + MTEST_VG_MEM_INIT(port, MPI_MAX_PORT_NAME * sizeof(char)); + + MPI_Comm_rank(intra_comm, &intra_rank); + MPI_Comm_size(intra_comm, &intra_nproc); + IF_VERBOSE(intra_rank == 0, ("%s: TEST mismatched accept case, %d nprocs in group\n", + (gid == SERVER_GID) ? "server" : "client", intra_nproc)); + + open_and_bcast_port(intra_rank, gid, &port); + + /* client group */ + if (gid == CLIENT_GID) { + int mpi_errno1 = MPI_SUCCESS, mpi_errno2 = MPI_SUCCESS; + + IF_VERBOSE(intra_rank == 0, ("client: connecting to <%s> with default timeout.\n", port)); + mpi_errno1 = MPI_Comm_connect(port, MPI_INFO_NULL, 0, intra_comm, &comm); + if (mpi_errno1 != MPI_SUCCESS) { + errs += check_errno(mpi_errno1, MPI_ERR_PORT); + } + + /* At least one of the connect calls should return MPI_ERR_PORT */ + IF_VERBOSE(intra_rank == 0, ("client: connecting to <%s> again.\n", port)); + mpi_errno2 = MPI_Comm_connect(port, MPI_INFO_NULL, 0, intra_comm, &comm); + if (mpi_errno1 == MPI_SUCCESS) { + errs += check_errno(mpi_errno2, MPI_ERR_PORT); + } + else { + errs += check_errno(mpi_errno2, MPI_SUCCESS); + } + } + else if (gid == SERVER_GID) { + /* NOTE: if accept hangs, try increase MPIR_CVAR_CH3_COMM_CONN_TIMEOUT. */ + IF_VERBOSE(intra_rank == 0, ("server: accepting connection to <%s>.\n", port)); + MPI_Comm_accept(port, MPI_INFO_NULL, 0, intra_comm, &comm); + } + + if (comm != MPI_COMM_NULL) { + IF_VERBOSE(intra_rank == 0, ("connection matched, freeing communicator.\n")); + MPI_Comm_free(&comm); + } + + if (intra_rank == 0 && gid == SERVER_GID) { + IF_VERBOSE(1, ("server root: closing port.\n")); + MPI_Close_port(port); + } + + return errs; +} + +int test_no_accept(MPI_Comm intra_comm, int gid) +{ + char port[MPI_MAX_PORT_NAME], timeout = 0; + MPI_Info info = MPI_INFO_NULL; + MPI_Comm comm = MPI_COMM_NULL; + int mpi_errno = MPI_SUCCESS; + int errs = 0; + int intra_rank, intra_nproc; + int server_rank = 0, local_server_rank = 0; + + MTEST_VG_MEM_INIT(port, MPI_MAX_PORT_NAME * sizeof(char)); + + MPI_Comm_rank(intra_comm, &intra_rank); + MPI_Comm_size(intra_comm, &intra_nproc); + IF_VERBOSE(intra_rank == 0, ("%s: TEST no accept case, %d nprocs in group\n", + (gid == SERVER_GID) ? "server" : "client", intra_nproc)); + + open_and_bcast_port(intra_rank, gid, &port); + + /* client group */ + if (gid == CLIENT_GID) { + IF_VERBOSE(intra_rank == 0, + ("client: case 1 - connecting to <%s> with default timeout.\n", port)); + mpi_errno = MPI_Comm_connect(port, MPI_INFO_NULL, 0, intra_comm, &comm); + errs += check_errno(mpi_errno, MPI_ERR_PORT); + + MPI_Info_create(&info); + MPI_Info_set(info, "timeout", TIMEOUT); + + IF_VERBOSE(intra_rank == 0, + ("client: case 2 - connecting to <%s> with specified timeout <%s>s.\n", port, + TIMEOUT)); + mpi_errno = MPI_Comm_connect(port, info, 0, intra_comm, &comm); + errs += check_errno(mpi_errno, MPI_ERR_PORT); + + MPI_Info_free(&info); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* server closes port after client finished connection */ + if (intra_rank == 0 && gid == SERVER_GID) { + IF_VERBOSE(1, ("server root: closing port.\n")); + MPI_Close_port(port); + } + + return errs; +} + + +int main(int argc, char *argv[]) +{ + MPI_Comm intra_comm = MPI_COMM_NULL; + int sub_rank, sub_nproc; + int errs = 0, allerrs = 0; + + if (getenv("MPITEST_VERBOSE")) { + verbose = 1; + } + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + if (nproc < 2) { + printf("At least two processes needed to run this test.\n"); + goto exit; + } + + /* Separate all processes into a server group and a client group. + * Processes in client group connect to server group. */ + MPI_Comm_split(MPI_COMM_WORLD, rank % 2, rank, &intra_comm); + MPI_Errhandler_set(intra_comm, MPI_ERRORS_RETURN); + + errs = RUN_TEST(intra_comm, rank % 2); + + MPI_Comm_free(&intra_comm); + MPI_Reduce(&errs, &allerrs, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + + if (rank == 0 && allerrs == 0) { + printf(" No Errors\n"); + fflush(stdout); + } + + exit: + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/errors/spawn/testlist.in b/test/mpi/errors/spawn/testlist.in index e8ba5e3..f685b06 100644 --- a/test/mpi/errors/spawn/testlist.in +++ b/test/mpi/errors/spawn/testlist.in @@ -1,3 +1,7 @@ badport 2 @namepub_tests@unpub 1 @namepub_tests@lookup_name 1 +connect_timeout_no_accept 2 env=MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT=20 +connect_timeout_mismatch 2 env=MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT=20 +connect_timeout_no_accept 5 env=MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT=20 +connect_timeout_mismatch 5 env=MPIR_CVAR_CH3_COMM_CONNECT_TIMEOUT=20 ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/channels/nemesis/netmod/tcp/socksm.c | 6 +- .../ch3/channels/nemesis/netmod/tcp/tcp_send.c | 6 +- src/mpid/ch3/errnames.txt | 3 + src/mpid/ch3/include/mpid_port.h | 200 +++++ src/mpid/ch3/include/mpidimpl.h | 35 +- src/mpid/ch3/include/mpidpkt.h | 19 + src/mpid/ch3/src/ch3u_handle_recv_pkt.c | 7 + src/mpid/ch3/src/ch3u_port.c | 874 ++++++++++++++++++-- src/mpid/ch3/src/mpid_finalize.c | 4 + src/mpid/ch3/src/mpid_port.c | 4 + test/mpi/errors/spawn/Makefile.am | 9 +- test/mpi/errors/spawn/connect_timeout.c | 225 +++++ test/mpi/errors/spawn/testlist.in | 4 + 13 files changed, 1298 insertions(+), 98 deletions(-) create mode 100644 src/mpid/ch3/include/mpid_port.h create mode 100644 test/mpi/errors/spawn/connect_timeout.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-421-g1c63de8
by noreply＠mpich.org 02 Sep '16

02 Sep '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 1c63de860856b416864625ef7be4b379e729d99a (commit) via 4159d8a7fd6d0d5e1efbea8cf1eb825222266fae (commit) from 072a6bf7da26b0fcc7ccbc91cea611eb1ca42a9a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/1c63de860856b416864625ef7be4b379e… commit 1c63de860856b416864625ef7be4b379e729d99a Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Tue Aug 30 10:26:34 2016 -0500 pm/util: Add configure check for time function We should not assume that time is checked for elsewhere in configure. Fixes a build issue with CH4 and alternative process managers like gforker. Signed-off-by: Lena Oden <loden(a)anl.gov> diff --git a/src/pm/util/subconfigure.m4 b/src/pm/util/subconfigure.m4 index ef985c3..16b308f 100644 --- a/src/pm/util/subconfigure.m4 +++ b/src/pm/util/subconfigure.m4 @@ -245,6 +245,7 @@ if test "$ac_cv_func_thread_policy_set" = yes ; then fi AC_CHECK_HEADERS([string.h sys/time.h time.h stdlib.h sys/socket.h wait.h errno.h]) +AC_CHECK_FUNCS(time) # Check for socklen_t . # (note the conditional inclusion of sys/socket.h) AC_CACHE_CHECK([whether socklen_t is defined (in sys/socket.h if present)], http://git.mpich.org/mpich.git/commitdiff/4159d8a7fd6d0d5e1efbea8cf1eb82522… commit 4159d8a7fd6d0d5e1efbea8cf1eb825222266fae Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Mon Aug 29 17:02:38 2016 -0500 CH4/nm: Remove datatype dup hook The dup hook was removed in [b752654a5e44]. Signed-off-by: Lena Oden <loden(a)anl.gov> diff --git a/src/mpid/ch4/netmod/include/netmod.h b/src/mpid/ch4/netmod/include/netmod.h index 6e3457d..e9d4e58 100644 --- a/src/mpid/ch4/netmod/include/netmod.h +++ b/src/mpid/ch4/netmod/include/netmod.h @@ -335,8 +335,6 @@ typedef int (*MPIDI_NM_iscatterv_t) (const void *sendbuf, const int *sendcounts, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPIR_Comm * comm_ptr, MPI_Request * req); -typedef void (*MPIDI_NM_type_dup_hook_t) (MPIR_Datatype * old_datatype_p, - MPIR_Datatype * new_datatype_p); typedef int (*MPIDI_NM_type_create_hook_t) (MPIR_Datatype * datatype_p); typedef int (*MPIDI_NM_type_free_hook_t) (MPIR_Datatype * datatype_p); typedef int (*MPIDI_NM_op_create_hook_t) (MPIR_Op * op_p); @@ -476,7 +474,6 @@ typedef struct MPIDI_NM_native_funcs { MPIDI_NM_iscatterv_t iscatterv; /* Datatype hooks */ MPIDI_NM_type_create_hook_t type_create_hook; - MPIDI_NM_type_dup_hook_t type_dup_hook; MPIDI_NM_type_free_hook_t type_free_hook; /* Op hooks */ MPIDI_NM_op_create_hook_t op_create_hook; @@ -965,9 +962,6 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_NM_iscatterv(const void *sendbuf, const int * MPI_Datatype recvtype, int root, MPIR_Comm * comm_ptr, MPI_Request * req) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX void MPIDI_NM_type_dup_hook(MPIR_Datatype * old_datatype_p, - MPIR_Datatype * - new_datatype_p) MPL_STATIC_INLINE_SUFFIX; MPL_STATIC_INLINE_PREFIX int MPIDI_NM_type_create_hook(MPIR_Datatype * datatype_p) MPL_STATIC_INLINE_SUFFIX; MPL_STATIC_INLINE_PREFIX int MPIDI_NM_type_free_hook(MPIR_Datatype * diff --git a/src/mpid/ch4/netmod/include/netmod_impl.h b/src/mpid/ch4/netmod/include/netmod_impl.h index 781da99..4ba34a1 100644 --- a/src/mpid/ch4/netmod/include/netmod_impl.h +++ b/src/mpid/ch4/netmod/include/netmod_impl.h @@ -963,12 +963,6 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_NM_iscatterv(const void *sendbuf, const int * recvcount, recvtype, root, comm_ptr, req); }; -MPL_STATIC_INLINE_PREFIX void MPIDI_NM_type_dup_hook(MPIR_Datatype * old_datatype_p, - MPIR_Datatype * new_datatype_p) -{ - return MPIDI_NM_native_func->type_dup_hook(old_datatype_p, new_datatype_p); -}; - MPL_STATIC_INLINE_PREFIX int MPIDI_NM_type_create_hook(MPIR_Datatype * datatype_p) { return MPIDI_NM_native_func->type_create_hook(datatype_p); diff --git a/src/mpid/ch4/netmod/ofi/func_table.c b/src/mpid/ch4/netmod/ofi/func_table.c index bed0f63..f1586d4 100644 --- a/src/mpid/ch4/netmod/ofi/func_table.c +++ b/src/mpid/ch4/netmod/ofi/func_table.c @@ -142,7 +142,6 @@ MPIDI_NM_native_funcs_t MPIDI_NM_native_ofi_funcs = { MPIDI_NM_iscatter, MPIDI_NM_iscatterv, MPIDI_NM_type_create_hook, - MPIDI_NM_type_dup_hook, MPIDI_NM_type_free_hook, MPIDI_NM_op_create_hook, MPIDI_NM_op_free_hook, diff --git a/src/mpid/ch4/netmod/ofi/ofi_datatype.h b/src/mpid/ch4/netmod/ofi/ofi_datatype.h index 8e110ca..7a12b23 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_datatype.h +++ b/src/mpid/ch4/netmod/ofi/ofi_datatype.h @@ -23,9 +23,4 @@ static inline int MPIDI_NM_type_create_hook(MPIR_Datatype * datatype_p) return 0; } -static inline void MPIDI_NM_type_dup_hook(MPIR_Datatype * old_datatype_p, - MPIR_Datatype * new_datatype_p) -{ - return; -} #endif diff --git a/src/mpid/ch4/netmod/portals4/func_table.c b/src/mpid/ch4/netmod/portals4/func_table.c index 8d39e10..59cfe2b 100644 --- a/src/mpid/ch4/netmod/portals4/func_table.c +++ b/src/mpid/ch4/netmod/portals4/func_table.c @@ -142,7 +142,6 @@ MPIDI_NM_native_funcs_t MPIDI_NM_native_portals4_funcs = { MPIDI_NM_iscatter, MPIDI_NM_iscatterv, MPIDI_NM_type_create_hook, - MPIDI_NM_type_dup_hook, MPIDI_NM_type_free_hook, MPIDI_NM_op_create_hook, MPIDI_NM_op_free_hook diff --git a/src/mpid/ch4/netmod/portals4/ptl_datatype.h b/src/mpid/ch4/netmod/portals4/ptl_datatype.h index 95f346a..8f1cc2e 100644 --- a/src/mpid/ch4/netmod/portals4/ptl_datatype.h +++ b/src/mpid/ch4/netmod/portals4/ptl_datatype.h @@ -19,10 +19,4 @@ static inline int MPIDI_NM_type_create_hook(MPIR_Datatype * datatype_p) return 0; } -static inline void MPIDI_NM_type_dup_hook(MPIR_Datatype * old_datatype_p, - MPIR_Datatype * new_datatype_p) -{ - return; -} - #endif /* NETMOD_PTL_DATATYPE_H_INCLUDED */ diff --git a/src/mpid/ch4/netmod/stubnm/globals.c b/src/mpid/ch4/netmod/stubnm/globals.c index 9dc545b..190d561 100644 --- a/src/mpid/ch4/netmod/stubnm/globals.c +++ b/src/mpid/ch4/netmod/stubnm/globals.c @@ -142,7 +142,6 @@ MPIDI_NM_native_funcs_t MPIDI_NM_native_stubnm_funcs = { MPIDI_NM_iscatter, MPIDI_NM_iscatterv, MPIDI_NM_type_create_hook, - MPIDI_NM_type_dup_hook, MPIDI_NM_type_free_hook, MPIDI_NM_op_create_hook, MPIDI_NM_op_free_hook, diff --git a/src/mpid/ch4/netmod/stubnm/stubnm_datatype.h b/src/mpid/ch4/netmod/stubnm/stubnm_datatype.h index 323aa6d..cac199f 100644 --- a/src/mpid/ch4/netmod/stubnm/stubnm_datatype.h +++ b/src/mpid/ch4/netmod/stubnm/stubnm_datatype.h @@ -25,11 +25,4 @@ static inline int MPIDI_NM_type_create_hook(MPIR_Datatype * datatype_p) return 0; } -static inline void MPIDI_NM_type_dup_hook(MPIR_Datatype * old_datatype_p, - MPIR_Datatype * new_datatype_p) -{ - MPIR_Assert(0); - return; -} - #endif diff --git a/src/mpid/ch4/netmod/ucx/func_table.c b/src/mpid/ch4/netmod/ucx/func_table.c index e4801eb..01c9ec5 100644 --- a/src/mpid/ch4/netmod/ucx/func_table.c +++ b/src/mpid/ch4/netmod/ucx/func_table.c @@ -140,7 +140,6 @@ MPIDI_NM_native_funcs_t MPIDI_NM_native_ucx_funcs = { MPIDI_NM_iscatter, MPIDI_NM_iscatterv, MPIDI_NM_type_create_hook, - MPIDI_NM_type_dup_hook, MPIDI_NM_type_free_hook, MPIDI_NM_op_create_hook, MPIDI_NM_op_free_hook diff --git a/src/mpid/ch4/netmod/ucx/ucx_datatype.h b/src/mpid/ch4/netmod/ucx/ucx_datatype.h index eb115f0..ce4f933 100644 --- a/src/mpid/ch4/netmod/ucx/ucx_datatype.h +++ b/src/mpid/ch4/netmod/ucx/ucx_datatype.h @@ -141,10 +141,4 @@ static inline int MPIDI_NM_type_create_hook(MPIR_Datatype * datatype_p) return 0; } -static inline void MPIDI_NM_type_dup_hook(MPIR_Datatype * old_datatype_p, - MPIR_Datatype * new_datatype_p) -{ - return; -} - #endif /* NETMOD_UCX_DATATYPE_H_INCLUDED */ ----------------------------------------------------------------------- Summary of changes: src/mpid/ch4/netmod/include/netmod.h | 6 ------ src/mpid/ch4/netmod/include/netmod_impl.h | 6 ------ src/mpid/ch4/netmod/ofi/func_table.c | 1 - src/mpid/ch4/netmod/ofi/ofi_datatype.h | 5 ----- src/mpid/ch4/netmod/portals4/func_table.c | 1 - src/mpid/ch4/netmod/portals4/ptl_datatype.h | 6 ------ src/mpid/ch4/netmod/stubnm/globals.c | 1 - src/mpid/ch4/netmod/stubnm/stubnm_datatype.h | 7 ------- src/mpid/ch4/netmod/ucx/func_table.c | 1 - src/mpid/ch4/netmod/ucx/ucx_datatype.h | 6 ------ src/pm/util/subconfigure.m4 | 1 + 11 files changed, 1 insertions(+), 40 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-419-g072a6bf
by noreply＠mpich.org 02 Sep '16

02 Sep '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 072a6bf7da26b0fcc7ccbc91cea611eb1ca42a9a (commit) from c5b844ad3397bc654258595dc8bd595b6738296e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/072a6bf7da26b0fcc7ccbc91cea611eb1… commit 072a6bf7da26b0fcc7ccbc91cea611eb1ca42a9a Author: Hajime Fujita <hajime.fujita(a)intel.com> Date: Wed Aug 24 10:44:43 2016 -0500 CH4/OFI: Specify FI_DELIVERY_COMPLETE for RMA FI_DELIVERY_COMPLETE flag has been missing when setting up TX context/issuing fi_*msg functions. This patch makes sure that RMA operations are issued with FI_DELIVERY_COMPLETE flag. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpid/ch4/netmod/ofi/ofi_init.h b/src/mpid/ch4/netmod/ofi/ofi_init.h index 31eceeb..9ea0a18 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_init.h +++ b/src/mpid/ch4/netmod/ofi/ofi_init.h @@ -305,6 +305,7 @@ static inline int MPIDI_OFI_init_generic(int rank, int ret; struct fi_tx_attr tx_attr; memset(&tx_attr, 0, sizeof(tx_attr)); + tx_attr.op_flags = FI_DELIVERY_COMPLETE | FI_COMPLETION; MPIDI_OFI_CALL_RETURN(fi_stx_context(MPIDI_Global.domain, &tx_attr, &MPIDI_Global.stx_ctx, NULL /* context */), ret); diff --git a/src/mpid/ch4/netmod/ofi/ofi_rma.h b/src/mpid/ch4/netmod/ofi/ofi_rma.h index fda3960..0675b18 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_rma.h +++ b/src/mpid/ch4/netmod/ofi/ofi_rma.h @@ -38,12 +38,12 @@ { \ MPIDI_OFI_REQUEST_CREATE((*(sigreq)), MPIR_REQUEST_KIND__RMA); \ MPIR_cc_set((*(sigreq))->cc_ptr, 0); \ - *(flags) = FI_COMPLETION; \ + *(flags) = FI_COMPLETION | FI_DELIVERY_COMPLETE; \ *(ep) = MPIDI_OFI_WIN(win).ep; \ } \ else { \ *(ep) = MPIDI_OFI_WIN(win).ep_nocmpl; \ - *(flags) = 0ULL; \ + *(flags) = FI_DELIVERY_COMPLETE; \ } \ } while (0) ----------------------------------------------------------------------- Summary of changes: src/mpid/ch4/netmod/ofi/ofi_init.h | 1 + src/mpid/ch4/netmod/ofi/ofi_rma.h | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-418-gc5b844a
by noreply＠mpich.org 31 Aug '16

31 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via c5b844ad3397bc654258595dc8bd595b6738296e (commit) from 637da5d8e537d70680fa1394a5fd3f988f2e5b2d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/c5b844ad3397bc654258595dc8bd595b6… commit c5b844ad3397bc654258595dc8bd595b6738296e Author: Min Si <msi(a)anl.gov> Date: Tue Aug 23 15:57:24 2016 -0500 Added a test to check RMAs issued over overlapping windows. Signed-off-by: Lena Oden <loden(a)anl.gov> diff --git a/test/mpi/rma/Makefile.am b/test/mpi/rma/Makefile.am index bf84531..7c8bfc7 100644 --- a/test/mpi/rma/Makefile.am +++ b/test/mpi/rma/Makefile.am @@ -169,6 +169,11 @@ noinst_PROGRAMS = \ win_shared_gacc_flush_load \ win_shared_fop_flush_load \ win_shared_cas_flush_load \ + overlap_wins_put \ + overlap_wins_acc \ + overlap_wins_gacc \ + overlap_wins_fop \ + overlap_wins_cas \ rget-unlock if BUILD_MPIX_TESTS @@ -277,3 +282,15 @@ win_shared_acc_flush_load_CPPFLAGS = -DTEST_ACC $(AM_CPPFLAGS) win_shared_gacc_flush_load_CPPFLAGS = -DTEST_GACC $(AM_CPPFLAGS) win_shared_fop_flush_load_CPPFLAGS = -DTEST_FOP $(AM_CPPFLAGS) win_shared_cas_flush_load_CPPFLAGS = -DTEST_CAS $(AM_CPPFLAGS) + +overlap_wins_put_SOURCES = overlap_wins_rma.c +overlap_wins_acc_SOURCES = overlap_wins_rma.c +overlap_wins_gacc_SOURCES = overlap_wins_rma.c +overlap_wins_fop_SOURCES = overlap_wins_rma.c +overlap_wins_cas_SOURCES = overlap_wins_rma.c + +overlap_wins_put_CPPFLAGS = -DTEST_PUT $(AM_CPPFLAGS) +overlap_wins_acc_CPPFLAGS = -DTEST_ACC $(AM_CPPFLAGS) +overlap_wins_gacc_CPPFLAGS = -DTEST_GACC $(AM_CPPFLAGS) +overlap_wins_fop_CPPFLAGS = -DTEST_FOP $(AM_CPPFLAGS) +overlap_wins_cas_CPPFLAGS = -DTEST_CAS $(AM_CPPFLAGS) diff --git a/test/mpi/rma/overlap_wins_rma.c b/test/mpi/rma/overlap_wins_rma.c new file mode 100644 index 0000000..9ccdeef --- /dev/null +++ b/test/mpi/rma/overlap_wins_rma.c @@ -0,0 +1,304 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2016 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <mpi.h> + +/* This test checks the remote completion of flush with RMA write-like operations + * (PUT, ACC, GET_ACC, FOP, CAS) concurrently issued from different origin processes + * to the same target over overlapping windows (i.e., two windows exposing the same + * memory region) + * 1. The first [nprocs-1] processes perform as origin, and the last process + * performs as target. + * 2. Everyone allocates a buffer and creates [nprocs-1] windows over the same buffer. + * 3. Every origin P[i] issues RMA operations and flush to target through + * wins[i] respectively, to different location winbuf[i] on target. + * 4. Finally, every origin P[i] issues GET and flush to obtain winbuf[i] on + * target and checks the correctness. */ + +#define ITER 10 +#define BUF_CNT 2 + +#if defined(TEST_CAS) +#define MPI_DATATYPE MPI_INT +#define DATATYPE int +#define DATATYPE_FORMAT "%d" +#else +#define MPI_DATATYPE MPI_DOUBLE +#define DATATYPE double +#define DATATYPE_FORMAT "%.1f" +#endif + +DATATYPE local_buf[BUF_CNT], result_buf[BUF_CNT], compare_buf[BUF_CNT]; +DATATYPE exp_target_val = 0.0; + +const int verbose = 0; + +int rank = -1, nprocs = 0; +int norigins, target; +MPI_Win *wins; +int win_size = 0, win_cnt = 0; + +DATATYPE *winbuf = NULL, *my_base = NULL; + +#define verbose_print(str,...) { \ + if (verbose) { \ + fprintf(stdout, str, ## __VA_ARGS__); \ + fflush(stdout); \ + } \ + } +#define error_print(str,...) { \ + fprintf(stderr, str, ## __VA_ARGS__); \ + fflush(stderr); \ + } + +/* Define operation name for error message */ +#ifdef TEST_PUT +const char *rma_name = "Put"; +#elif defined(TEST_ACC) +const char *rma_name = "Accumulate"; +#elif defined(TEST_GACC) +const char *rma_name = "Get_accumulate"; +#elif defined(TEST_FOP) +const char *rma_name = "Fetch_and_op"; +#elif defined(TEST_CAS) +const char *rma_name = "Compare_and_swap"; +#else +const char *rma_name = "None"; +#endif + +/* Issue functions for different RMA operations */ +#ifdef TEST_PUT +static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr /* NULL */ , + DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp, + MPI_Win win) +{ + MPI_Put(origin_addr, 1, MPI_DATATYPE, dst, target_disp, 1, MPI_DATATYPE, win); +} +#elif defined(TEST_ACC) +static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr /* NULL */ , + DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp, + MPI_Win win) +{ + MPI_Accumulate(origin_addr, 1, MPI_DATATYPE, dst, target_disp, 1, MPI_DATATYPE, MPI_SUM, win); +} +#elif defined(TEST_GACC) +static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr, + DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp, + MPI_Win win) +{ + MPI_Get_accumulate(origin_addr, 1, MPI_DATATYPE, result_addr, 1, MPI_DATATYPE, dst, target_disp, + 1, MPI_DATATYPE, MPI_SUM, win); +} +#elif defined(TEST_FOP) +static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr, + DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp, + MPI_Win win) +{ + MPI_Fetch_and_op(origin_addr, result_addr, MPI_DATATYPE, dst, target_disp, MPI_SUM, win); +} +#elif defined(TEST_CAS) +static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr, + DATATYPE * compare_addr, int dst, MPI_Aint target_disp, MPI_Win win) +{ + MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, MPI_DATATYPE, dst, target_disp, + win); +} +#else +#define issue_rma_op(loc_addr, result_addr, compare_addr, dst, target_disp, win) +#endif + +static inline void set_iteration_data(int x) +{ + int i; + +#if defined(TEST_CAS) + for (i = 0; i < BUF_CNT; i++) + compare_buf[i] = local_buf[i]; /* always equal, thus swap happens */ +#endif + + for (i = 0; i < BUF_CNT; i++) { + local_buf[i] = rank + i + x; + +#if defined(TEST_CAS) || defined(TEST_PUT) + exp_target_val = local_buf[i]; /* swap */ +#else + exp_target_val += local_buf[i]; /* sum */ +#endif + } +} + +static void print_origin_data(void) +{ + int i; + + printf("[%d] local_buf: ", rank); + for (i = 0; i < BUF_CNT; i++) + printf(DATATYPE_FORMAT " ", local_buf[i]); + printf("\n"); + + printf("[%d] result_buf: ", rank); + for (i = 0; i < BUF_CNT; i++) + printf(DATATYPE_FORMAT " ", result_buf[i]); + printf("\n"); +} + +static void print_target_data(void) +{ + int i; + printf("[%d] winbuf: ", rank); + for (i = 0; i < win_cnt; i++) + printf(DATATYPE_FORMAT " ", winbuf[i]); + printf("\n"); + fflush(stdout); +} + +static int run_test() +{ + int errors = 0; + int i, x; + int dst = 0, target_disp = 0; + MPI_Win win = MPI_WIN_NULL; + DATATYPE target_val = 0.0; + + /* 1. Specify working window and displacement. + * - Target: no RMA issued, always check results on wins[0]. + * - Origins: issue RMA on different window and different memory location */ + if (rank == target) { + win = wins[0]; + } + else { + win = wins[rank]; + target_disp = rank; + } + dst = target; + + /* 2. Every one resets local data */ + memset(local_buf, 0, sizeof(local_buf)); + memset(result_buf, 0, sizeof(result_buf)); + memset(compare_buf, 0, sizeof(compare_buf)); + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank != target) { + + /* 3. Origins issue RMA to target over its working window */ + MPI_Win_lock(MPI_LOCK_SHARED, dst, 0, win); + verbose_print("[%d] RMA start, test %s (dst=%d, target_disp=%d, win 0x%x) - flush\n", + rank, rma_name, dst, target_disp, win); + + for (x = 0; x < ITER; x++) { + /* update local buffers and expected value in every iteration */ + set_iteration_data(x); + + for (i = 0; i < BUF_CNT; i++) + issue_rma_op(&local_buf[i], &result_buf[i], &compare_buf[i], dst, target_disp, win); + MPI_Win_flush(dst, win); + + if (verbose) + print_origin_data(); + } + + /* 4. Check correctness of final target value */ + MPI_Get(&target_val, 1, MPI_DATATYPE, dst, target_disp, 1, MPI_DATATYPE, win); + MPI_Win_flush(dst, win); + if (target_val != exp_target_val) { + error_print("rank %d (iter %d) - check %s, got target_val = " + DATATYPE_FORMAT ", expected " DATATYPE_FORMAT "\n", rank, x, + rma_name, target_val, exp_target_val); + errors++; + } + + MPI_Win_unlock(dst, win); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* 5. Every one prints window buffer */ + if (verbose && rank == target) { + MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); + print_target_data(); + MPI_Win_unlock(rank, win); + } + + return errors; +} + +static void init_windows(void) +{ + int i = 0; + + /* Everyone creates norigins overlapping windows. */ + winbuf = malloc(win_size); + memset(winbuf, 0, win_size); + + wins = malloc(norigins * sizeof(MPI_Win)); + for (i = 0; i < norigins; i++) { + wins[i] = MPI_WIN_NULL; + MPI_Win_create(winbuf, win_size, sizeof(DATATYPE), MPI_INFO_NULL, MPI_COMM_WORLD, &wins[i]); + } +} + +static void destroy_windows(void) +{ + int i = 0; + for (i = 0; i < norigins; i++) { + if (wins[i] != MPI_WIN_NULL) + MPI_Win_free(&wins[i]); + } + free(wins); + free(winbuf); +} + +int main(int argc, char *argv[]) +{ + int errors = 0, all_errors = 0; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + if (nprocs < 3) { + if (rank == 0) { + error_print("Error: must use at least 3 processes\n"); + } + MPI_Barrier(MPI_COMM_WORLD); + MPI_Abort(MPI_COMM_WORLD, 1); + } + + /* The last rank performs as target, all others are origin. + * Every origin accesses to a different memory location on the target. */ + target = nprocs - 1; + norigins = nprocs - 1; + win_cnt = nprocs - 1; + win_size = sizeof(DATATYPE) * win_cnt; + + if (rank == 0) { + verbose_print("[%d] %d origins, target rank = %d\n", rank, norigins, target); + } + + init_windows(); + MPI_Barrier(MPI_COMM_WORLD); + + /* start test */ + errors = run_test(); + + MPI_Barrier(MPI_COMM_WORLD); + MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + + if (rank == 0 && all_errors == 0) { + fprintf(stdout, " No Errors\n"); + fflush(stdout); + } + + destroy_windows(); + MPI_Finalize(); + + return 0; +} diff --git a/test/mpi/rma/testlist.in b/test/mpi/rma/testlist.in index b6ee786..5ba6df1 100644 --- a/test/mpi/rma/testlist.in +++ b/test/mpi/rma/testlist.in @@ -158,6 +158,11 @@ gacc_flush_get 3 mpiversion=3.0 fop_flush_get 3 mpiversion=3.0 cas_flush_get 3 mpiversion=3.0 rget-unlock 2 mpiversion=3.0 +overlap_wins_put 3 +overlap_wins_acc 3 +overlap_wins_gacc 3 +overlap_wins_fop 3 +overlap_wins_cas 3 ## This test is not strictly correct. This was meant to test out the ## case when MPI_Test is not nonblocking. However, we ended up ----------------------------------------------------------------------- Summary of changes: test/mpi/rma/Makefile.am | 17 +++ test/mpi/rma/overlap_wins_rma.c | 304 +++++++++++++++++++++++++++++++++++++++ test/mpi/rma/testlist.in | 5 + 3 files changed, 326 insertions(+), 0 deletions(-) create mode 100644 test/mpi/rma/overlap_wins_rma.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-417-g637da5d
by noreply＠mpich.org 31 Aug '16

31 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 637da5d8e537d70680fa1394a5fd3f988f2e5b2d (commit) from bfefee88130f464b8037b9250a6394d1f7a14dc6 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/637da5d8e537d70680fa1394a5fd3f988… commit 637da5d8e537d70680fa1394a5fd3f988f2e5b2d Author: Yanfei Guo <yguo(a)anl.gov> Date: Wed Aug 31 16:59:44 2016 -0500 CH4: Fix mlut create for intercomm_merge Old code is incorrectly optimized for intercomm_merge. No reviewer. diff --git a/src/mpid/ch4/src/ch4i_comm.h b/src/mpid/ch4/src/ch4i_comm.h index 3e9f6a5..d837115 100644 --- a/src/mpid/ch4/src/ch4i_comm.h +++ b/src/mpid/ch4/src/ch4i_comm.h @@ -950,22 +950,11 @@ static inline int MPIDII_set_map(MPIDII_rank_map_t * src_rmap, MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE, (MPL_DBG_FDEST, " check map_size %d, src_comm_size %d", MPIDII_map_size(*mapper), src_comm_size)); - if (src_mode == MPIDII_RANK_MAP_MLUT) { - MPIDII_src_comm_to_mlut(src_rmap, dest_rmap, src_comm_size, - total_mapper_size, mapper_offset); - } - else { /* src_mode != MPIDII_RANK_MAP_MLUT */ - MPIDII_src_comm_to_lut(src_rmap, dest_rmap, src_comm_size, - total_mapper_size, mapper_offset); - } + MPIDII_src_comm_to_mlut(src_rmap, dest_rmap, src_comm_size, + total_mapper_size, mapper_offset); } else { /* mapper->type == MPIR_COMM_MAP_TYPE__IRREGULAR */ - if (src_mode == MPIDII_RANK_MAP_MLUT) { - MPIDII_src_mlut_to_mlut(src_rmap, dest_rmap, mapper, total_mapper_size, mapper_offset); - } - else { /* src_mode != MPIDII_RANK_MAP_MLUT */ - MPIDII_src_map_to_lut(src_rmap, dest_rmap, mapper, total_mapper_size, mapper_offset); - } + MPIDII_src_mlut_to_mlut(src_rmap, dest_rmap, mapper, total_mapper_size, mapper_offset); } fn_exit: ----------------------------------------------------------------------- Summary of changes: src/mpid/ch4/src/ch4i_comm.h | 17 +++-------------- 1 files changed, 3 insertions(+), 14 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-416-gbfefee8
by noreply＠mpich.org 31 Aug '16

31 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via bfefee88130f464b8037b9250a6394d1f7a14dc6 (commit) from fa897516e38b52e3f91abe2da7197630d7d2c639 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/bfefee88130f464b8037b9250a6394d1f… commit bfefee88130f464b8037b9250a6394d1f7a14dc6 Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Thu Aug 4 12:38:01 2016 -0500 ROMIO: Remove outdated configure check Now that MPL is embedded in ROMIO, there is no need to check for MPICH memory allocation routines. Remove the configure check and use the MPL versions directly. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/confdb/aclocal_romio.m4 b/confdb/aclocal_romio.m4 index 64091ba..94f4e2e 100644 --- a/confdb/aclocal_romio.m4 +++ b/confdb/aclocal_romio.m4 @@ -786,28 +786,6 @@ EOF fi rm -f conftest$EXEEXT mpitest.c ])dnl -define(PAC_TEST_MPIU_FUNCS,[ - AC_MSG_CHECKING(support for MPICH memory macros) - rm -f mpitest.c - cat > mpitest.c <<EOF -#include "mpi.h" -#include "stdio.h" - main(Int argc, char **argv) - { - MPL_free(NULL); - } -EOF - rm -f conftest$EXEEXT - $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1 - if test -x conftest$EXEEXT ; then - AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined]) - else - AC_MSG_RESULT(no) - fi - rm -f conftest$EXEEXT mpitest.c -])dnl -dnl define(PAC_TEST_MPI_GREQUEST,[ AC_MSG_CHECKING(support for generalized requests) rm -f mpitest.c diff --git a/src/mpi/romio/adio/common/malloc.c b/src/mpi/romio/adio/common/malloc.c index f0838e1..b10d75f 100644 --- a/src/mpi/romio/adio/common/malloc.c +++ b/src/mpi/romio/adio/common/malloc.c @@ -37,12 +37,6 @@ void *ADIOI_Calloc_fn(size_t nelem, size_t elsize, int lineno, const char *fname void *ADIOI_Realloc_fn(void *ptr, size_t size, int lineno, const char *fname); void ADIOI_Free_fn(void *ptr, int lineno, const char *fname); -/* we used to call MPIU_Malloc, MPIU_Calloc, etc. these macros either call - * malloc/calloc directly or they add instrumentation/tracing. When we re-did - * the build system in 2011 we hacked ROMIO to call MPIU_trmalloc directly -- - * once again ROMIO's sort-of standalone status bites us. We dont' test for - * MPIU_FUNCS: these are set in configure with an AC_DEFINE when we figure out - * we are built as part of MPICH */ void *ADIOI_Malloc_fn(size_t size, int lineno, const char *fname) { void *new; @@ -50,11 +44,7 @@ void *ADIOI_Malloc_fn(size_t size, int lineno, const char *fname) #ifdef ROMIO_XFS new = (void *) memalign(XFS_MEMALIGN, size); #else -#ifdef HAVE_MPIU_FUNCS new = (void *) MPL_malloc(size); -#else - new = (void *) malloc(size); -#endif #endif if (!new && size) { FPRINTF(stderr, "Out of memory in file %s, line %d\n", fname, lineno); @@ -70,11 +60,7 @@ void *ADIOI_Calloc_fn(size_t nelem, size_t elsize, int lineno, const char *fname { void *new; -#ifdef HAVE_MPIU_FUNCS new = (void *) MPL_calloc(nelem, elsize); -#else - new = (void *) calloc(nelem, elsize); -#endif if (!new && nelem) { FPRINTF(stderr, "Out of memory in file %s, line %d\n", fname, lineno); MPI_Abort(MPI_COMM_WORLD, 1); @@ -88,11 +74,7 @@ void *ADIOI_Realloc_fn(void *ptr, size_t size, int lineno, const char *fname) { void *new; -#ifdef HAVE_MPIU_FUNCS new = (void *) MPL_realloc(ptr, size); -#else - new = (void *) realloc(ptr, size); -#endif if (!new && size) { FPRINTF(stderr, "realloc failed in file %s, line %d\n", fname, lineno); MPI_Abort(MPI_COMM_WORLD, 1); @@ -110,11 +92,7 @@ void ADIOI_Free_fn(void *ptr, int lineno, const char *fname) MPI_Abort(MPI_COMM_WORLD, 1); } -#ifdef HAVE_MPIU_FUNCS MPL_free(ptr); -#else - free(ptr); -#endif } diff --git a/src/mpi/romio/configure.ac b/src/mpi/romio/configure.ac index 67732c4..eaec4f2 100644 --- a/src/mpi/romio/configure.ac +++ b/src/mpi/romio/configure.ac @@ -1516,7 +1516,6 @@ elif test $FROM_MPICH = yes ; then AC_DEFINE(HAVE_STATUS_SET_BYTES,1,[Define if status_set_bytes available]) DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST 1" DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#define HAVE_MPI_GREQUEST_EXTENSIONS 1" - AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined]) AC_DEFINE(HAVE_MPIX_H, 1, []) AC_DEFINE(HAVE_MPIIO_CONST, const, Set if MPI-IO prototypes use const qualifier) AC_DEFINE(HAVE_MPI_TYPE_SIZE_X, 1, [Define if MPI library provides MPI_TYPE_SIZE_X]) @@ -1531,7 +1530,6 @@ fi if test $WITHIN_KNOWN_MPI_IMPL = no ; then PAC_TEST_MPIR_STATUS_SET_BYTES PAC_TEST_MPI_GREQUEST - PAC_TEST_MPIU_FUNCS AC_DEFINE(PRINT_ERR_MSG,1,[Define for printing error messages]) AC_CHECK_TYPE([MPI_Count],[],[AC_DEFINE_UNQUOTED([MPI_Count],[MPI_Aint],[Define to "MPI_Aint" if MPI does not provide MPI_Count]) ], [[#include <mpi.h>]]) PAC_TEST_NEEDS_CONST ----------------------------------------------------------------------- Summary of changes: confdb/aclocal_romio.m4 | 22 ---------------------- src/mpi/romio/adio/common/malloc.c | 22 ---------------------- src/mpi/romio/configure.ac | 2 -- 3 files changed, 0 insertions(+), 46 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-415-gfa89751
by noreply＠mpich.org 31 Aug '16

31 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via fa897516e38b52e3f91abe2da7197630d7d2c639 (commit) via 3d1e1a3943bd33184a48517ed87c4dd1883edce4 (commit) from 8d4de3bd8d64fb545e9046e6b78ef766a90aaee9 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/fa897516e38b52e3f91abe2da7197630d… commit fa897516e38b52e3f91abe2da7197630d7d2c639 Author: Yanfei Guo <yguo(a)anl.gov> Date: Thu Aug 25 14:11:14 2016 -0500 CH4: fix getting pid from local_map No reviewer. diff --git a/src/mpid/ch4/src/ch4r_proc.h b/src/mpid/ch4/src/ch4r_proc.h index 65c75bf..91ccfbe 100644 --- a/src/mpid/ch4/src/ch4r_proc.h +++ b/src/mpid/ch4/src/ch4r_proc.h @@ -149,8 +149,8 @@ static inline int MPIDIU_comm_rank_to_pid_local(MPIR_Comm * comm, int rank, int break; case MPIDII_RANK_MAP_STRIDE: case MPIDII_RANK_MAP_STRIDE_INTRA: - *index = MPIDII_CALC_STRIDE_SIMPLE(rank, MPIDII_COMM(comm, map).reg.stride.stride, - MPIDII_COMM(comm, map).reg.stride.offset); + *index = MPIDII_CALC_STRIDE_SIMPLE(rank, MPIDII_COMM(comm, local_map).reg.stride.stride, + MPIDII_COMM(comm, local_map).reg.stride.offset); break; case MPIDII_RANK_MAP_STRIDE_BLOCK: case MPIDII_RANK_MAP_STRIDE_BLOCK_INTRA: http://git.mpich.org/mpich.git/commitdiff/3d1e1a3943bd33184a48517ed87c4dd18… commit 3d1e1a3943bd33184a48517ed87c4dd1883edce4 Author: Yanfei Guo <yguo(a)anl.gov> Date: Thu Aug 25 14:07:06 2016 -0500 CH4: fix debug print out in rank map No reviewer. diff --git a/src/mpid/ch4/src/ch4i_comm.h b/src/mpid/ch4/src/ch4i_comm.h index a5575d7..3e9f6a5 100644 --- a/src/mpid/ch4/src/ch4i_comm.h +++ b/src/mpid/ch4/src/ch4i_comm.h @@ -177,7 +177,7 @@ static inline int MPIDII_detect_regular_model(int *lpid, int size, if (st < 0 || st <= bs) { return MPIDII_SRC_MAPPER_IRREGULAR; } - MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE, (MPL_DBG_FDEST, "\tdetect model: stride %d", bs)); + MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE, (MPL_DBG_FDEST, "\tdetect model: stride %d", st)); for (i = bs; i < size; i++) { if (lpid[i] != MPIDII_CALC_STRIDE(i, st, bs, off)) { return MPIDII_SRC_MAPPER_IRREGULAR; @@ -873,9 +873,9 @@ static inline int MPIDII_check_convert_lut_to_regular(MPIDII_rank_map_t * src) MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE, (MPL_DBG_FDEST, "\toffset: %d", src->reg.stride.offset)); MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE, - (MPL_DBG_FDEST, "\toffset: %d", src->reg.stride.blocksize)); + (MPL_DBG_FDEST, "\tblocksize: %d", src->reg.stride.blocksize)); MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE, - (MPL_DBG_FDEST, "\toffset: %d", src->reg.stride.stride)); + (MPL_DBG_FDEST, "\tstride: %d", src->reg.stride.stride)); break; } fn_exit: ----------------------------------------------------------------------- Summary of changes: src/mpid/ch4/src/ch4i_comm.h | 6 +++--- src/mpid/ch4/src/ch4r_proc.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-413-g8d4de3b
by noreply＠mpich.org 31 Aug '16

31 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 8d4de3bd8d64fb545e9046e6b78ef766a90aaee9 (commit) from ac947d5c4576ff9683eb945c823c0974173a7b4b (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/8d4de3bd8d64fb545e9046e6b78ef766a… commit 8d4de3bd8d64fb545e9046e6b78ef766a90aaee9 Author: Eric Chamberland <ericc(a)giref.ulaval.ca> Date: Mon Aug 29 09:53:13 2016 -0500 binding/cxx: Fix extra semicolon warning Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/binding/cxx/buildiface b/src/binding/cxx/buildiface index ab0b1a4..2491f4d 100755 --- a/src/binding/cxx/buildiface +++ b/src/binding/cxx/buildiface @@ -3573,14 +3573,14 @@ sub Clone_method { MPI_Comm_dup( (MPI_Comm)the_real_comm, &ncomm); Comm *clone = new $classname(ncomm); return *clone; - }; + } #else virtual $classname & Clone(void) const { MPI_Comm ncomm; MPI_Comm_dup( (MPI_Comm)the_real_comm, &ncomm); $classname *clone = new $classname(ncomm); return *clone; - }; + } #endif\n"; } sub Comm_methods { @@ -3673,12 +3673,12 @@ sub Nullcomm_methods { virtual Comm & Clone(void) const { Comm *clone = new Nullcomm(MPI_COMM_NULL); return *clone; - }; + } #else virtual Nullcomm & Clone(void) const { Nullcomm *clone = new Nullcomm(); return *clone; - }; + } #endif\n"; } ----------------------------------------------------------------------- Summary of changes: src/binding/cxx/buildiface | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-412-gac947d5
by noreply＠mpich.org 31 Aug '16

31 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via ac947d5c4576ff9683eb945c823c0974173a7b4b (commit) from 1e51b91103ffb101586135c3ac49d4c0b53b6c2d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/ac947d5c4576ff9683eb945c823c09741… commit ac947d5c4576ff9683eb945c823c0974173a7b4b Author: Wesley Bland <wesley.bland(a)intel.com> Date: Tue Jul 26 09:08:23 2016 -0500 CH4/OFI: Fix AV_TABLE mode Fixes csr/mpich-opa#387 Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpid/ch4/netmod/ofi/ofi_impl.h b/src/mpid/ch4/netmod/ofi/ofi_impl.h index cd80910..79aec52 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_impl.h +++ b/src/mpid/ch4/netmod/ofi/ofi_impl.h @@ -22,16 +22,10 @@ #define MPIDI_OFI_COMM(comm) ((comm)->dev.ch4.netmod.ofi) #define MPIDI_OFI_COMM_TO_INDEX(comm,rank) \ MPIDIU_comm_rank_to_pid(comm, rank, NULL, NULL) -#ifdef MPIDI_OFI_CONFIG_USE_AV_TABLE -#define MPIDI_OFI_COMM_TO_PHYS(comm,rank) \ - ((fi_addr_t)MPIDI_OFI_COMM_TO_INDEX(comm,rank)) -#define MPIDI_OFI_TO_PHYS(avtid, rank) ((fi_addr_t)rank) -#else #define MPIDI_OFI_COMM_TO_PHYS(comm,rank) \ MPIDI_OFI_AV(MPIDIU_comm_rank_to_av((comm), (rank))).dest #define MPIDI_OFI_TO_PHYS(avtid, lpid) \ MPIDI_OFI_AV(&MPIDIU_get_av((avtid), (lpid))).dest -#endif #define MPIDI_OFI_WIN(win) ((win)->dev.netmod.ofi) /* diff --git a/src/mpid/ch4/netmod/ofi/ofi_init.h b/src/mpid/ch4/netmod/ofi/ofi_init.h index d8a9579..31eceeb 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_init.h +++ b/src/mpid/ch4/netmod/ofi/ofi_init.h @@ -285,12 +285,11 @@ static inline int MPIDI_OFI_init_generic(int rank, if (do_av_table) { av_attr.type = FI_AV_TABLE; - mapped_table = NULL; } else { av_attr.type = FI_AV_MAP; - mapped_table = (fi_addr_t *) MPL_malloc(size * sizeof(fi_addr_t)); } + mapped_table = (fi_addr_t *) MPL_malloc(size * sizeof(fi_addr_t)); av_attr.rx_ctx_bits = MPIDI_OFI_MAX_ENDPOINTS_BITS; @@ -373,12 +372,10 @@ static inline int MPIDI_OFI_init_generic(int rank, /* Table is constructed. Map it */ /* -------------------------------- */ MPIDI_OFI_CALL(fi_av_insert(MPIDI_Global.av, table, size, mapped_table, 0ULL, NULL), avmap); - if (!do_av_table) { /* AV_MAP */ - for (i = 0; i < size; i++) { - MPIDI_OFI_AV(&MPIDIU_get_av(0, i)).dest = mapped_table[i]; - } - MPL_free(mapped_table); + for (i = 0; i < size; i++) { + MPIDI_OFI_AV(&MPIDIU_get_av(0, i)).dest = mapped_table[i]; } + MPL_free(mapped_table); /* -------------------------------- */ /* Create the id to object maps */ diff --git a/src/mpid/ch4/netmod/ofi/ofi_pre.h b/src/mpid/ch4/netmod/ofi/ofi_pre.h index 5e2e966..8d3d690 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_pre.h +++ b/src/mpid/ch4/netmod/ofi/ofi_pre.h @@ -179,9 +179,7 @@ typedef struct { } MPIDI_OFI_gpid_t; typedef struct { -#ifndef MPIDI_OFI_CONFIG_USE_AV_TABLE fi_addr_t dest; -#endif #ifdef MPIDI_OFI_CONFIG_USE_SCALABLE_ENDPOINTS unsigned ep_idx:MPIDI_OFI_MAX_ENDPOINTS_BITS; #endif ----------------------------------------------------------------------- Summary of changes: src/mpid/ch4/netmod/ofi/ofi_impl.h | 6 ------ src/mpid/ch4/netmod/ofi/ofi_init.h | 11 ++++------- src/mpid/ch4/netmod/ofi/ofi_pre.h | 2 -- 3 files changed, 4 insertions(+), 15 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2-411-g1e51b91
by noreply＠mpich.org 30 Aug '16

30 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 1e51b91103ffb101586135c3ac49d4c0b53b6c2d (commit) from ee2f29508c93d5b37a979c0b0820a5401b6b69f8 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/1e51b91103ffb101586135c3ac49d4c0b… commit 1e51b91103ffb101586135c3ac49d4c0b53b6c2d Author: Hajime Fujita <hajime.fujita(a)intel.com> Date: Fri Aug 5 16:04:10 2016 -0500 CH4/OFI: Use regular EP for RMA instead of alias It turns out that using an endpoint from fi_ep_alias would incur additional redirection. This patch eliminates the use of fi_ep_alias in RMA window creation to avoid this potential cost. Fixes csr/mpich-opa#394 Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpid/ch4/netmod/ofi/ofi_win.h b/src/mpid/ch4/netmod/ofi/ofi_win.h index 2975845..2bb045d 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_win.h +++ b/src/mpid/ch4/netmod/ofi/ofi_win.h @@ -154,12 +154,14 @@ static inline int MPIDI_OFI_win_init_generic(MPI_Aint length, MPIR_Assert(finfo); finfo->ep_attr->tx_ctx_cnt = FI_SHARED_CONTEXT; /* Request a shared context */ MPIDI_OFI_CALL_RETURN(fi_endpoint(MPIDI_Global.domain, - finfo, &MPIDI_OFI_WIN(win).ep, NULL), ret); - fi_freeinfo(finfo); + finfo, + &MPIDI_OFI_WIN(win).ep, + NULL), ret); if (ret < 0) { MPL_DBG_MSG(MPIDI_CH4_DBG_GENERAL, VERBOSE, "Failed to create per-window EP (with completion), " "falling back to global EP/counter scheme"); + fi_freeinfo(finfo); goto fallback_global; } @@ -180,8 +182,11 @@ static inline int MPIDI_OFI_win_init_generic(MPI_Aint length, FI_READ | FI_WRITE), bind); MPIDI_OFI_CALL(fi_ep_bind(MPIDI_OFI_WIN(win).ep, &MPIDI_Global.av->fid, 0), bind); - MPIDI_OFI_CALL_RETURN(fi_ep_alias(MPIDI_OFI_WIN(win).ep, &MPIDI_OFI_WIN(win).ep_nocmpl, - FI_TRANSMIT), ret); + MPIDI_OFI_CALL_RETURN(fi_endpoint(MPIDI_Global.domain, + finfo, + &MPIDI_OFI_WIN(win).ep_nocmpl, + NULL), ret); + fi_freeinfo(finfo); if (ret < 0) { MPL_DBG_MSG(MPIDI_CH4_DBG_GENERAL, VERBOSE, "Failed to create an EP alias, " @@ -191,6 +196,14 @@ static inline int MPIDI_OFI_win_init_generic(MPI_Aint length, goto fallback_global; } + MPIDI_OFI_CALL(fi_ep_bind(MPIDI_OFI_WIN(win).ep_nocmpl, + &MPIDI_Global.stx_ctx->fid, 0), bind); + MPIDI_OFI_CALL(fi_ep_bind(MPIDI_OFI_WIN(win).ep_nocmpl, + &MPIDI_OFI_WIN(win).cmpl_cntr->fid, + FI_READ | FI_WRITE), bind); + MPIDI_OFI_CALL(fi_ep_bind(MPIDI_OFI_WIN(win).ep_nocmpl, + &MPIDI_Global.av->fid, 0), bind); + MPIDI_OFI_CALL(fi_enable(MPIDI_OFI_WIN(win).ep), ep_enable); MPIDI_OFI_CALL(fi_enable(MPIDI_OFI_WIN(win).ep_nocmpl), ep_enable); } ----------------------------------------------------------------------- Summary of changes: src/mpid/ch4/netmod/ofi/ofi_win.h | 21 +++++++++++++++++---- 1 files changed, 17 insertions(+), 4 deletions(-) hooks/post-receive -- MPICH primary repository

1 0