April 2015 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.2b1-60-g79e2398
by noreply＠mpich.org 10 Apr '15

10 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 79e239819cd638f7e0fc2c6c9b597ab3b685805b (commit) from 5addea2c60bd4b00568e624491d327dc3675a5c0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/79e239819cd638f7e0fc2c6c9b597ab3b… commit 79e239819cd638f7e0fc2c6c9b597ab3b685805b Author: Charles J Archer <charles.j.archer(a)intel.com> Date: Fri Apr 10 10:11:37 2015 -0700 Simplification of OFI initialization flow using allocinfo diff --git a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c index 7291219..5024bfa 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ofi/ofi_init.c @@ -18,7 +18,7 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_ { int ret, fi_version, i, len, pmi_errno; int mpi_errno = MPI_SUCCESS; - info_t hints, *prov_tagged, *prov_use; + info_t *hints, *prov_tagged, *prov_use; cq_attr_t cq_attr; av_attr_t av_attr; char kvsname[OFI_KVSAPPSTRLEN], key[OFI_KVSAPPSTRLEN], bc[OFI_KVSAPPSTRLEN]; @@ -51,12 +51,12 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_ /* We expect to register all memory up front for use with this */ /* endpoint, so the netmod requires dynamic memory regions */ /* ------------------------------------------------------------------------ */ - memset(&hints, 0, sizeof(hints)); - hints.mode = FI_CONTEXT; - hints.caps = FI_TAGGED; /* Tag matching interface */ - hints.caps |= FI_CANCEL; /* Support cancel */ - hints.caps |= FI_DYNAMIC_MR; /* Global dynamic mem region */ - + hints = fi_allocinfo(); + hints->mode = FI_CONTEXT; + hints->ep_attr->type = FI_EP_RDM; /* Reliable datagram */ + hints->caps = FI_TAGGED; /* Tag matching interface */ + hints->caps |= FI_CANCEL; /* Support cancel */ + hints->caps |= FI_DYNAMIC_MR; /* Global dynamic mem region */ /* ------------------------------------------------------------------------ */ /* FI_VERSION provides binary backward and forward compatibility support */ @@ -70,29 +70,15 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_ /* remote node or service. this does not necessarily allocate resources. */ /* Pass NULL for name/service because we want a list of providers supported */ /* ------------------------------------------------------------------------ */ - domain_attr_t domain_attr; - memset(&domain_attr, 0, sizeof(domain_attr)); - - ep_attr_t ep_attr; - memset(&ep_attr, 0, sizeof(ep_attr)); - - tx_attr_t tx_attr; - memset(&tx_attr, 0, sizeof(tx_attr)); - - hints.ep_attr = &ep_attr; - hints.ep_attr->type = FI_EP_RDM; - - domain_attr.threading = FI_THREAD_ENDPOINT; - domain_attr.control_progress = FI_PROGRESS_AUTO; - domain_attr.data_progress = FI_PROGRESS_AUTO; - hints.domain_attr = &domain_attr; - hints.tx_attr = &tx_attr; + hints->domain_attr->threading = FI_THREAD_ENDPOINT; + hints->domain_attr->control_progress = FI_PROGRESS_AUTO; + hints->domain_attr->data_progress = FI_PROGRESS_AUTO; FI_RC(fi_getinfo(fi_version, /* Interface version requested */ NULL, /* Optional name or fabric to resolve */ NULL, /* Service name or port number to request */ 0ULL, /* Flag: node/service specify local address */ - &hints, /* In: Hints to filter available providers */ + hints, /* In: Hints to filter available providers */ &prov_tagged), /* Out: List of providers that match hints */ getinfo); MPIU_ERR_CHKANDJUMP4(prov_tagged == NULL, mpi_errno, MPI_ERR_OTHER, @@ -187,6 +173,7 @@ int MPID_nem_ofi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_ /* --------------------------- */ /* Free providers info */ /* --------------------------- */ + fi_freeinfo(hints); fi_freeinfo(prov_use); /* ---------------------------------------------------- */ ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/netmod/ofi/ofi_init.c | 37 ++++++------------- 1 files changed, 12 insertions(+), 25 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-59-g5addea2
by noreply＠mpich.org 10 Apr '15

10 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 5addea2c60bd4b00568e624491d327dc3675a5c0 (commit) via e01a20b6e78b73788b26a3a62fc676ae11c6e439 (commit) via be82b6a78f23e3f550e7461682587bef13d60ee0 (commit) via e7eab9df185a2a0075e177a915fa1d269fe840b2 (commit) from 08c52c2e268bc4721eaba289bef121ccc67cfe58 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/5addea2c60bd4b00568e624491d327dc3… commit 5addea2c60bd4b00568e624491d327dc3675a5c0 Author: Pavan Balaji <balaji(a)anl.gov> Date: Thu Apr 9 12:27:42 2015 -0500 Update .gitignore. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/test/mpi/.gitignore b/test/mpi/.gitignore index 8020972..0886a79 100644 --- a/test/mpi/.gitignore +++ b/test/mpi/.gitignore @@ -101,8 +101,9 @@ /coll/scatterv /coll/scattern /coll/rerr -/coll/bcast2 -/coll/bcast3 +/coll/bcast_full +/coll/bcast_min_datatypes +/coll/bcast_comm_world /coll/exscan2 /coll/exscan /coll/icallgather http://git.mpich.org/mpich.git/commitdiff/e01a20b6e78b73788b26a3a62fc676ae1… commit e01a20b6e78b73788b26a3a62fc676ae11c6e439 Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Apr 8 18:55:34 2015 -0500 Simplify the bcast test. The current number of combinations we are checking are too many, causing the test to take too long on some platforms. This patch simplifies the test, so we build two versions of the test. In the first version, we run only on COMM_WORLD but go through all datatypes. In the second version, we run on all communicators, but go through only a small subset of datatypes. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/test/mpi/coll/Makefile.am b/test/mpi/coll/Makefile.am index 471f52f..0ab3079 100644 --- a/test/mpi/coll/Makefile.am +++ b/test/mpi/coll/Makefile.am @@ -31,7 +31,9 @@ noinst_PROGRAMS = \ alltoallw1 \ alltoallw2 \ alltoallw_zeros \ - bcast \ + bcast_full \ + bcast_min_datatypes \ + bcast_comm_world \ bcasttest \ bcastzerotype \ coll2 \ @@ -108,6 +110,14 @@ noinst_PROGRAMS = \ allgatherv4_LDADD = $(LDADD) -lm +bcast_full_SOURCES = bcast.c + +bcast_min_datatypes_CPPFLAGS = -DBCAST_MIN_DATATYPES_ONLY $(AM_CPPFLAGS) +bcast_min_datatypes_SOURCES = bcast.c + +bcast_comm_world_CPPFLAGS = -DBCAST_COMM_WORLD_ONLY $(AM_CPPFLAGS) +bcast_comm_world_SOURCES = bcast.c + # Create tests of the nonblocking collectives by reusing the blocking tests, # using the MPI profiling interface to intercept the blocking call and # replace it with a MPI_Ixxx/MPI_Wait pair. diff --git a/test/mpi/coll/bcast.c b/test/mpi/coll/bcast.c index 85a51c1..993bdfb 100644 --- a/test/mpi/coll/bcast.c +++ b/test/mpi/coll/bcast.c @@ -30,6 +30,11 @@ int main(int argc, char *argv[]) if (comm == MPI_COMM_NULL) continue; +#if defined BCAST_COMM_WORLD_ONLY + if (comm != MPI_COMM_WORLD) + continue; +#endif /* BCAST_COMM_WORLD_ONLY */ + /* Determine the sender and receiver */ MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); @@ -42,9 +47,9 @@ int main(int argc, char *argv[]) /* To shorten test time, only run the default version of datatype tests * for comm world and run the minimum version for other communicators. */ - if (comm != MPI_COMM_WORLD) { - MTestInitMinDatatypes(); - } +#if defined BCAST_MIN_DATATYPES_ONLY + MTestInitMinDatatypes(); +#endif /* BCAST_MIN_DATATYPES_ONLY */ while (MTestGetDatatypes(&sendtype, &recvtype, count)) { for (root = 0; root < size; root++) { diff --git a/test/mpi/coll/testlist b/test/mpi/coll/testlist index 832b505..61ad2a3 100644 --- a/test/mpi/coll/testlist +++ b/test/mpi/coll/testlist @@ -32,10 +32,11 @@ allgatherv4 4 timeLimit=600 bcasttest 4 bcasttest 8 bcasttest 10 -bcast 4 +bcast_full 4 # More that 8 processes are required to get bcast to switch to the long # msg algorithm (see coll definitions in mpiimpl.h) -bcast 10 timeLimit=1500 +bcast_min_datatypes 10 timeLimit=1200 +bcast_comm_world 10 timeLimit=1200 bcastzerotype 1 bcastzerotype 4 bcastzerotype 5 http://git.mpich.org/mpich.git/commitdiff/be82b6a78f23e3f550e7461682587bef1… commit be82b6a78f23e3f550e7461682587bef13d60ee0 Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Apr 8 18:44:00 2015 -0500 Cosmetic changes to the bcast2 test. 1. Renamed bcast2 to bcast. 2. White-space cleanup for bcast.c Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/test/mpi/coll/Makefile.am b/test/mpi/coll/Makefile.am index 28ac6e4..471f52f 100644 --- a/test/mpi/coll/Makefile.am +++ b/test/mpi/coll/Makefile.am @@ -31,7 +31,7 @@ noinst_PROGRAMS = \ alltoallw1 \ alltoallw2 \ alltoallw_zeros \ - bcast2 \ + bcast \ bcasttest \ bcastzerotype \ coll2 \ diff --git a/test/mpi/coll/bcast.c b/test/mpi/coll/bcast.c new file mode 100644 index 0000000..85a51c1 --- /dev/null +++ b/test/mpi/coll/bcast.c @@ -0,0 +1,86 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2003 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ +#include "mpi.h" +#include <stdio.h> +#include "mpitest.h" + +/* +static char MTEST_Descrip[] = "Test of broadcast with various roots and datatypes"; +*/ + +int main(int argc, char *argv[]) +{ + int errs = 0, err; + int rank, size, root; + int minsize = 2, count; + MPI_Comm comm; + MTestDatatype sendtype, recvtype; + + MTest_Init(&argc, &argv); + + /* The following illustrates the use of the routines to + * run through a selection of communicators and datatypes. + * Use subsets of these for tests that do not involve combinations + * of communicators, datatypes, and counts of datatypes */ + while (MTestGetIntracommGeneral(&comm, minsize, 1)) { + if (comm == MPI_COMM_NULL) + continue; + + /* Determine the sender and receiver */ + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + /* To improve reporting of problems about operations, we + * change the error handler to errors return */ + MPI_Errhandler_set(comm, MPI_ERRORS_RETURN); + + MTEST_DATATYPE_FOR_EACH_COUNT(count) { + + /* To shorten test time, only run the default version of datatype tests + * for comm world and run the minimum version for other communicators. */ + if (comm != MPI_COMM_WORLD) { + MTestInitMinDatatypes(); + } + + while (MTestGetDatatypes(&sendtype, &recvtype, count)) { + for (root = 0; root < size; root++) { + if (rank == root) { + sendtype.InitBuf(&sendtype); + err = MPI_Bcast(sendtype.buf, sendtype.count, + sendtype.datatype, root, comm); + if (err) { + errs++; + MTestPrintError(err); + } + } + else { + recvtype.InitBuf(&recvtype); + err = MPI_Bcast(recvtype.buf, recvtype.count, + recvtype.datatype, root, comm); + if (err) { + errs++; + fprintf(stderr, "Error with communicator %s and datatype %s\n", + MTestGetIntracommName(), MTestGetDatatypeName(&recvtype)); + MTestPrintError(err); + } + err = MTestCheckRecv(0, &recvtype); + if (err) { + errs += errs; + } + } + } + MTestFreeDatatype(&recvtype); + MTestFreeDatatype(&sendtype); + } + } + MTestFreeComm(&comm); + } + + MTest_Finalize(errs); + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/coll/bcast2.c b/test/mpi/coll/bcast2.c deleted file mode 100644 index 40054cd..0000000 --- a/test/mpi/coll/bcast2.c +++ /dev/null @@ -1,86 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * - * (C) 2003 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ -#include "mpi.h" -#include <stdio.h> -#include "mpitest.h" - -/* -static char MTEST_Descrip[] = "Test of broadcast with various roots and datatypes"; -*/ - -int main( int argc, char *argv[] ) -{ - int errs = 0, err; - int rank, size, root; - int minsize = 2, count; - MPI_Comm comm; - MTestDatatype sendtype, recvtype; - - MTest_Init( &argc, &argv ); - - /* The following illustrates the use of the routines to - run through a selection of communicators and datatypes. - Use subsets of these for tests that do not involve combinations - of communicators, datatypes, and counts of datatypes */ - while (MTestGetIntracommGeneral( &comm, minsize, 1 )) { - if (comm == MPI_COMM_NULL) continue; - - /* Determine the sender and receiver */ - MPI_Comm_rank( comm, &rank ); - MPI_Comm_size( comm, &size ); - - /* To improve reporting of problems about operations, we - change the error handler to errors return */ - MPI_Errhandler_set( comm, MPI_ERRORS_RETURN ); - - MTEST_DATATYPE_FOR_EACH_COUNT(count) { - - /* To shorten test time, only run the default version of datatype tests - * for comm world and run the minimum version for other communicators. */ - if (comm != MPI_COMM_WORLD) { - MTestInitMinDatatypes(); - } - - while (MTestGetDatatypes( &sendtype, &recvtype, count )) { - for (root=0; root<size; root++) { - if (rank == root) { - sendtype.InitBuf( &sendtype ); - err = MPI_Bcast( sendtype.buf, sendtype.count, - sendtype.datatype, root, comm ); - if (err) { - errs++; - MTestPrintError( err ); - } - } - else { - recvtype.InitBuf( &recvtype ); - err = MPI_Bcast( recvtype.buf, recvtype.count, - recvtype.datatype, root, comm ); - if (err) { - errs++; - fprintf( stderr, "Error with communicator %s and datatype %s\n", - MTestGetIntracommName(), - MTestGetDatatypeName( &recvtype ) ); - MTestPrintError( err ); - } - err = MTestCheckRecv( 0, &recvtype ); - if (err) { - errs += errs; - } - } - } - MTestFreeDatatype( &recvtype ); - MTestFreeDatatype( &sendtype ); - } - } - MTestFreeComm( &comm ); - } - - MTest_Finalize( errs ); - MPI_Finalize(); - return 0; -} diff --git a/test/mpi/coll/testlist b/test/mpi/coll/testlist index 24f6a91..832b505 100644 --- a/test/mpi/coll/testlist +++ b/test/mpi/coll/testlist @@ -32,10 +32,10 @@ allgatherv4 4 timeLimit=600 bcasttest 4 bcasttest 8 bcasttest 10 -bcast2 4 +bcast 4 # More that 8 processes are required to get bcast to switch to the long # msg algorithm (see coll definitions in mpiimpl.h) -bcast2 10 timeLimit=1500 +bcast 10 timeLimit=1500 bcastzerotype 1 bcastzerotype 4 bcastzerotype 5 http://git.mpich.org/mpich.git/commitdiff/e7eab9df185a2a0075e177a915fa1d269… commit e7eab9df185a2a0075e177a915fa1d269fe840b2 Author: Pavan Balaji <balaji(a)anl.gov> Date: Wed Apr 8 18:42:25 2015 -0500 Get rid of bcast3.c This test is exactly the same as bcast2. Originally these two tests were different, but over time they have become essentially the same. There's no point testing the same thing twice. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/test/mpi/coll/Makefile.am b/test/mpi/coll/Makefile.am index 98264a4..28ac6e4 100644 --- a/test/mpi/coll/Makefile.am +++ b/test/mpi/coll/Makefile.am @@ -32,7 +32,6 @@ noinst_PROGRAMS = \ alltoallw2 \ alltoallw_zeros \ bcast2 \ - bcast3 \ bcasttest \ bcastzerotype \ coll2 \ diff --git a/test/mpi/coll/bcast3.c b/test/mpi/coll/bcast3.c deleted file mode 100644 index 2da4bf8..0000000 --- a/test/mpi/coll/bcast3.c +++ /dev/null @@ -1,82 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * - * (C) 2003 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ -#include "mpi.h" -#include <stdio.h> -#include "mpitest.h" - -/* -static char MTEST_Descrip[] = "Test of broadcast with various roots and datatypes and sizes that are not powers of two"; -*/ - -int main( int argc, char *argv[] ) -{ - int errs = 0, err; - int rank, size, root; - int minsize = 2, count; - MPI_Comm comm; - MTestDatatype sendtype, recvtype; - - MTest_Init( &argc, &argv ); - - /* The following illustrates the use of the routines to - run through a selection of communicators and datatypes. - Use subsets of these for tests that do not involve combinations - of communicators, datatypes, and counts of datatypes */ - while (MTestGetIntracommGeneral( &comm, minsize, 1 )) { - if (comm == MPI_COMM_NULL) continue; - /* Determine the sender and receiver */ - MPI_Comm_rank( comm, &rank ); - MPI_Comm_size( comm, &size ); - - count = 1; - MTEST_DATATYPE_FOR_EACH_COUNT(count) { - - /* To shorten test time, only run the default version of datatype tests - * for comm world and run the minimum version for other communicators. */ - if (comm != MPI_COMM_WORLD) { - MTestInitMinDatatypes(); - } - - while (MTestGetDatatypes( &sendtype, &recvtype, count )) { - for (root=0; root<size; root++) { - if (rank == root) { - sendtype.InitBuf( &sendtype ); - err = MPI_Bcast( sendtype.buf, sendtype.count, - sendtype.datatype, root, comm ); - if (err) { - errs++; - MTestPrintError( err ); - } - } - else { - recvtype.InitBuf( &recvtype ); - err = MPI_Bcast( recvtype.buf, recvtype.count, - recvtype.datatype, root, comm ); - if (err) { - errs++; - fprintf( stderr, "Error with communicator %s and datatype %s\n", - MTestGetIntracommName(), - MTestGetDatatypeName( &recvtype ) ); - MTestPrintError( err ); - } - err = MTestCheckRecv( 0, &recvtype ); - if (err) { - errs += errs; - } - } - } - MTestFreeDatatype( &recvtype ); - MTestFreeDatatype( &sendtype ); - } - } - MTestFreeComm( &comm ); - } - - MTest_Finalize( errs ); - MPI_Finalize(); - return 0; -} diff --git a/test/mpi/coll/testlist b/test/mpi/coll/testlist index f776bc5..24f6a91 100644 --- a/test/mpi/coll/testlist +++ b/test/mpi/coll/testlist @@ -36,7 +36,6 @@ bcast2 4 # More that 8 processes are required to get bcast to switch to the long # msg algorithm (see coll definitions in mpiimpl.h) bcast2 10 timeLimit=1500 -bcast3 10 timeLimit=1500 bcastzerotype 1 bcastzerotype 4 bcastzerotype 5 ----------------------------------------------------------------------- Summary of changes: test/mpi/.gitignore | 5 +- test/mpi/coll/Makefile.am | 13 +++++- test/mpi/coll/bcast.c | 91 +++++++++++++++++++++++++++++++++++++++++++++ test/mpi/coll/bcast2.c | 86 ------------------------------------------ test/mpi/coll/bcast3.c | 82 ---------------------------------------- test/mpi/coll/testlist | 6 +- 6 files changed, 108 insertions(+), 175 deletions(-) create mode 100644 test/mpi/coll/bcast.c delete mode 100644 test/mpi/coll/bcast2.c delete mode 100644 test/mpi/coll/bcast3.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-55-g08c52c2
by noreply＠mpich.org 09 Apr '15

09 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 08c52c2e268bc4721eaba289bef121ccc67cfe58 (commit) from 0f96bf4d4da93d32d267ca16290d2ed194d1527c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/08c52c2e268bc4721eaba289bef121ccc… commit 08c52c2e268bc4721eaba289bef121ccc67cfe58 Author: Antonio Pena Monferrer <apenya(a)mcs.anl.gov> Date: Thu Apr 9 11:14:16 2015 -0500 Fix several RMA + async failures The datatype size was checked outside the appropriate branches in a couple of places Signed-off-by: Xin Zhao <xinzhao3(a)illinois.edu> diff --git a/src/mpid/ch3/src/ch3u_rma_pkthandler.c b/src/mpid/ch3/src/ch3u_rma_pkthandler.c index 2c72406..adfe90b 100644 --- a/src/mpid/ch3/src/ch3u_rma_pkthandler.c +++ b/src/mpid/ch3/src/ch3u_rma_pkthandler.c @@ -190,7 +190,6 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPIDI_CH3_Pkt_put_t *put_pkt = &pkt->put; MPID_Request *req = NULL; - MPI_Aint type_size; int complete = 0; char *data_buf = NULL; MPIDI_msg_sz_t data_len; @@ -217,13 +216,14 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } - MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); - if (pkt->type == MPIDI_CH3_PKT_PUT_IMMED) { + MPI_Aint type_size; /* Immed packet type is used when target datatype is predefined datatype. */ MPIU_Assert(MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)); + MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); + /* copy data from packet header to target buffer */ MPIU_Memcpy(put_pkt->addr, put_pkt->info.data, put_pkt->count * type_size); @@ -254,9 +254,13 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutRecvComplete; if (MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)) { + MPI_Aint type_size; + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RECV); req->dev.datatype = put_pkt->datatype; + MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); + req->dev.recv_data_sz = type_size * put_pkt->count; MPIU_Assert(req->dev.recv_data_sz > 0); @@ -772,7 +776,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_Win *win_ptr; int acquire_lock_fail = 0; int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size; MPI_Aint stream_elem_count, rest_len, total_len; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETACCUMULATE); @@ -794,8 +797,6 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } - MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); - if (pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) { size_t len; void *src = NULL, *dest = NULL; @@ -804,6 +805,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_CH3_Pkt_get_accum_resp_t *get_accum_resp_pkt = &upkt.get_accum_resp; MPID_IOV iov[MPID_IOV_LIMIT]; int iovcnt; + MPI_Aint type_size; *buflen = sizeof(MPIDI_CH3_Pkt_t); *rreqp = NULL; @@ -825,6 +827,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, win_ptr->at_completion_counter++; /* Calculate the length of reponse data, ensure that it fits into immed packet. */ + MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t); MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED); @@ -899,6 +902,8 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, data_buf = (char *) pkt + sizeof(MPIDI_CH3_Pkt_t); if (MPIR_DATATYPE_IS_PREDEFINED(get_accum_pkt->datatype)) { + MPI_Aint type_size; + MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_ACCUM_RECV); req->dev.datatype = get_accum_pkt->datatype; @@ -920,6 +925,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, req->dev.user_buf = req->dev.tmpbuf; + MPID_Datatype_get_size_macro(get_accum_pkt->datatype, type_size); total_len = type_size * get_accum_pkt->count; rest_len = total_len - req->dev.stream_offset; stream_elem_count = MPIDI_CH3U_SRBuf_size / extent; ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/src/ch3u_rma_pkthandler.c | 18 ++++++++++++------ 1 files changed, 12 insertions(+), 6 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-54-g0f96bf4
by noreply＠mpich.org 08 Apr '15

08 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 0f96bf4d4da93d32d267ca16290d2ed194d1527c (commit) from 70f23321889fd23c2bf3b3e4a8d72d5951ae11d0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/0f96bf4d4da93d32d267ca16290d2ed19… commit 0f96bf4d4da93d32d267ca16290d2ed194d1527c Author: Antonio J. Pena <apenya(a)mcs.anl.gov> Date: Wed Apr 8 22:17:10 2015 -0500 Revert "Fix unsafe datatype release in several RMA tests" This reverts commit b47d95f7a04a0fdbb82b4b5bf4a64ddb5c4b48aa. diff --git a/test/mpi/rma/put_base.c b/test/mpi/rma/put_base.c index a88f1eb..ba95a1c 100644 --- a/test/mpi/rma/put_base.c +++ b/test/mpi/rma/put_base.c @@ -36,12 +36,6 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; - MPI_Aint idx_loc[SUB_YDIM]; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - void *base_ptr; - MPI_Aint base_int; MTest_Init(&argc, &argv); @@ -64,35 +58,42 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - base_ptr = dst_buf; + /* Perform ITERATIONS strided put operations */ + + for (i = 0; i < ITERATIONS; i++) { + MPI_Aint idx_loc[SUB_YDIM]; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; - MPI_Get_address(base_ptr, &base_int); + void *base_ptr = dst_buf; + MPI_Aint base_int; - for (j = 0; j < SUB_YDIM; j++) { + MPI_Get_address(base_ptr, &base_int); + + for (j = 0; j < SUB_YDIM; j++) { MPI_Get_address(&src_buf[j*XDIM], &idx_loc[j]); idx_loc[j] = idx_loc[j] - base_int; idx_rem[j] = j*XDIM*sizeof(double); blk_len[j] = SUB_XDIM*sizeof(double); - } + } - MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); - MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); + MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); + MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided put operations */ - for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(base_ptr, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/put_bottom.c b/test/mpi/rma/put_bottom.c index 0bd0b7f..6634ea0 100644 --- a/test/mpi/rma/put_bottom.c +++ b/test/mpi/rma/put_bottom.c @@ -34,10 +34,6 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; - MPI_Aint idx_loc[SUB_YDIM]; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -58,30 +54,36 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - for (j = 0; j < SUB_YDIM; j++) { + /* Perform ITERATIONS strided put operations */ + + for (i = 0; i < ITERATIONS; i++) { + MPI_Aint idx_loc[SUB_YDIM]; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + + for (j = 0; j < SUB_YDIM; j++) { MPI_Get_address(&src_buf[j*XDIM], &idx_loc[j]); idx_rem[j] = j*XDIM*sizeof(double); blk_len[j] = SUB_XDIM*sizeof(double); - } + } - MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); - MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); + MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); + MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided put operations */ - for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(MPI_BOTTOM, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_acc_indexed.c b/test/mpi/rma/strided_acc_indexed.c index 45254c2..ac54f52 100644 --- a/test/mpi/rma/strided_acc_indexed.c +++ b/test/mpi/rma/strided_acc_indexed.c @@ -32,10 +32,6 @@ int main(int argc, char **argv) { int itr, i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf; MPI_Win buf_win; - MPI_Aint idx_loc[SUB_YDIM]; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -55,24 +51,30 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - for (i = 0; i < SUB_YDIM; i++) { + /* Perform ITERATIONS strided accumulate operations */ + + for (itr = 0; itr < ITERATIONS; itr++) { + MPI_Aint idx_loc[SUB_YDIM]; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + + for (i = 0; i < SUB_YDIM; i++) { MPI_Get_address(&src_buf[i*XDIM], &idx_loc[i]); idx_rem[i] = i*XDIM; blk_len[i] = SUB_XDIM; - } + } #ifdef ABSOLUTE - MPI_Type_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_DOUBLE, &src_type); + MPI_Type_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_DOUBLE, &src_type); #else - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); #endif - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided accumulate operations */ - for (itr = 0; itr < ITERATIONS; itr++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); #ifdef ABSOLUTE @@ -82,13 +84,13 @@ int main(int argc, char **argv) { #endif MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_acc_subarray.c b/test/mpi/rma/strided_acc_subarray.c index 385c928..c8f850c 100644 --- a/test/mpi/rma/strided_acc_subarray.c +++ b/test/mpi/rma/strided_acc_subarray.c @@ -32,14 +32,6 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf; MPI_Win buf_win; - int ndims = 2; - int src_arr_sizes[2] = { XDIM, YDIM }; - int src_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; - int src_arr_starts[2] = { 0, 0 }; - int dst_arr_sizes[2] = { XDIM, YDIM }; - int dst_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; - int dst_arr_starts[2] = { 0, 0 }; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -59,27 +51,39 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - MPI_Type_create_subarray(ndims, src_arr_sizes, src_arr_subsizes, src_arr_starts, + /* Perform ITERATIONS strided accumulate operations */ + + for (i = 0; i < ITERATIONS; i++) { + int ndims = 2; + int src_arr_sizes[2] = { XDIM, YDIM }; + int src_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; + int src_arr_starts[2] = { 0, 0 }; + int dst_arr_sizes[2] = { XDIM, YDIM }; + int dst_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; + int dst_arr_starts[2] = { 0, 0 }; + MPI_Datatype src_type, dst_type; + + MPI_Type_create_subarray(ndims, src_arr_sizes, src_arr_subsizes, src_arr_starts, MPI_ORDER_C, MPI_DOUBLE, &src_type); - MPI_Type_create_subarray(ndims, dst_arr_sizes, dst_arr_subsizes, dst_arr_starts, + MPI_Type_create_subarray(ndims, dst_arr_sizes, dst_arr_subsizes, dst_arr_starts, MPI_ORDER_C, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided accumulate operations */ - for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); + MPI_Accumulate(src_buf, 1, src_type, peer, 0, 1, dst_type, MPI_SUM, buf_win); + MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_get_indexed.c b/test/mpi/rma/strided_get_indexed.c index 37effd5..3a98d29 100644 --- a/test/mpi/rma/strided_get_indexed.c +++ b/test/mpi/rma/strided_get_indexed.c @@ -78,11 +78,11 @@ int main(int argc, char **argv) { MPI_Win_unlock(peer, buf_win); - MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&loc_type); MPI_Type_free(&rem_type); + MPI_Barrier(MPI_COMM_WORLD); + /* Verify that the results are correct */ errors = 0; diff --git a/test/mpi/rma/strided_getacc_indexed.c b/test/mpi/rma/strided_getacc_indexed.c index 2d52e5b..e3293a1 100644 --- a/test/mpi/rma/strided_getacc_indexed.c +++ b/test/mpi/rma/strided_getacc_indexed.c @@ -32,9 +32,6 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -55,19 +52,24 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - for (j = 0; j < SUB_YDIM; j++) { + /* Perform ITERATIONS strided accumulate operations */ + + for (i = 0; i < ITERATIONS; i++) { + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided accumulate operations */ - for (i = 0; i < ITERATIONS; i++) { /* PUT */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, @@ -79,13 +81,13 @@ int main(int argc, char **argv) { MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, 1, dst_type, MPI_NO_OP, buf_win); MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_getacc_indexed_shared.c b/test/mpi/rma/strided_getacc_indexed_shared.c index a67fdff..e1c8169 100644 --- a/test/mpi/rma/strided_getacc_indexed_shared.c +++ b/test/mpi/rma/strided_getacc_indexed_shared.c @@ -34,9 +34,6 @@ int main(int argc, char **argv) { double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; MPI_Comm shr_comm; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -65,19 +62,24 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - for (j = 0; j < SUB_YDIM; j++) { + /* Perform ITERATIONS strided accumulate operations */ + + for (i = 0; i < ITERATIONS; i++) { + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided accumulate operations */ - for (i = 0; i < ITERATIONS; i++) { /* PUT */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, @@ -89,13 +91,13 @@ int main(int argc, char **argv) { MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, 1, dst_type, MPI_NO_OP, buf_win); MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_putget_indexed.c b/test/mpi/rma/strided_putget_indexed.c index 677e196..09f17ae 100644 --- a/test/mpi/rma/strided_putget_indexed.c +++ b/test/mpi/rma/strided_putget_indexed.c @@ -32,9 +32,6 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -55,19 +52,24 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - for (j = 0; j < SUB_YDIM; j++) { + /* Perform ITERATIONS strided accumulate operations */ + + for (i = 0; i < ITERATIONS; i++) { + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided accumulate operations */ - for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(src_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); @@ -75,13 +77,13 @@ int main(int argc, char **argv) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get(dst_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_putget_indexed_shared.c b/test/mpi/rma/strided_putget_indexed_shared.c index 2d4f02e..1e65f2a 100644 --- a/test/mpi/rma/strided_putget_indexed_shared.c +++ b/test/mpi/rma/strided_putget_indexed_shared.c @@ -34,9 +34,6 @@ int main(int argc, char **argv) { double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; MPI_Comm shr_comm; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -65,19 +62,24 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - for (j = 0; j < SUB_YDIM; j++) { + /* Perform ITERATIONS strided accumulate operations */ + + for (i = 0; i < ITERATIONS; i++) { + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); - /* Perform ITERATIONS strided accumulate operations */ - for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(src_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); @@ -85,13 +87,13 @@ int main(int argc, char **argv) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get(dst_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); + + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); } MPI_Barrier(shr_comm); - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); - /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); ----------------------------------------------------------------------- Summary of changes: test/mpi/rma/put_base.c | 39 +++++++++++++------------ test/mpi/rma/put_bottom.c | 32 +++++++++++---------- test/mpi/rma/strided_acc_indexed.c | 34 ++++++++++++---------- test/mpi/rma/strided_acc_subarray.c | 38 ++++++++++++++----------- test/mpi/rma/strided_get_indexed.c | 4 +- test/mpi/rma/strided_getacc_indexed.c | 30 ++++++++++--------- test/mpi/rma/strided_getacc_indexed_shared.c | 30 ++++++++++--------- test/mpi/rma/strided_putget_indexed.c | 30 ++++++++++--------- test/mpi/rma/strided_putget_indexed_shared.c | 30 ++++++++++--------- 9 files changed, 142 insertions(+), 125 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-53-g70f2332
by noreply＠mpich.org 08 Apr '15

08 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 70f23321889fd23c2bf3b3e4a8d72d5951ae11d0 (commit) from 3ccad54bd508c2a48525f4204a1b1622c8c0bca9 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/70f23321889fd23c2bf3b3e4a8d72d595… commit 70f23321889fd23c2bf3b3e4a8d72d5951ae11d0 Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Wed Apr 1 13:00:14 2015 -0500 portals4: overhaul MPICH control message handling The previous design for MPICH control messages utilized a small set of "use once" buffers that could be quickly exhausted. The new approach processes all control messages via an unexpected queue. Benefits are a larger incoming message capacity, leading to less flow-control events. Signed-off-by: Antonio J. Pena <apenya(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c index 47bc252..39f8d8f 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c @@ -8,7 +8,8 @@ #include <mpl_utlist.h> #include "rptl.h" -#define NUM_RECV_BUFS 50 +#define NUM_RECV_BUFS 8 +#define BUFSIZE (1024*1024) #define CTL_TAG 0 #define GET_TAG 1 #define PAYLOAD_SIZE (PTL_MAX_EAGER - sizeof(MPIDI_CH3_Pkt_t)) @@ -16,8 +17,8 @@ #define SENDBUF(req_) REQ_PTL(req_)->chunk_buffer[0] #define TMPBUF(req_) REQ_PTL(req_)->chunk_buffer[1] -static char recvbufs[NUM_RECV_BUFS * PTL_MAX_EAGER]; -static ptl_me_t mes[NUM_RECV_BUFS]; +static char *recvbufs; +static ptl_me_t overflow_me; static ptl_handle_me_t me_handles[NUM_RECV_BUFS]; #undef FUNCNAME @@ -29,28 +30,29 @@ int MPID_nem_ptl_nm_init(void) int mpi_errno = MPI_SUCCESS; int i; int ret; - ptl_process_t id_any; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_NM_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_NM_INIT); /* init recv */ - id_any.phys.pid = PTL_PID_ANY; - id_any.phys.nid = PTL_NID_ANY; - + overflow_me.length = BUFSIZE; + overflow_me.ct_handle = PTL_CT_NONE; + overflow_me.uid = PTL_UID_ANY; + overflow_me.options = ( PTL_ME_OP_PUT | PTL_ME_MANAGE_LOCAL | PTL_ME_NO_TRUNCATE | PTL_ME_MAY_ALIGN | + PTL_ME_IS_ACCESSIBLE | PTL_ME_EVENT_LINK_DISABLE ); + overflow_me.match_id.phys.pid = PTL_PID_ANY; + overflow_me.match_id.phys.nid = PTL_NID_ANY; + overflow_me.match_bits = NPTL_MATCH(CTL_TAG, 0, MPI_ANY_SOURCE); + overflow_me.ignore_bits = NPTL_MATCH_IGNORE; + overflow_me.min_free = PTL_MAX_EAGER; + + /* allocate all overflow space at once */ + recvbufs = MPIU_Malloc(NUM_RECV_BUFS * BUFSIZE); + for (i = 0; i < NUM_RECV_BUFS; ++i) { - mes[i].start = &recvbufs[i * PTL_MAX_EAGER]; - mes[i].length = PTL_MAX_EAGER; - mes[i].ct_handle = PTL_CT_NONE; - mes[i].uid = PTL_UID_ANY; - mes[i].options = (PTL_ME_OP_PUT | PTL_ME_USE_ONCE | PTL_ME_EVENT_UNLINK_DISABLE | - PTL_ME_EVENT_LINK_DISABLE | PTL_ME_IS_ACCESSIBLE); - mes[i].match_id = id_any; - mes[i].match_bits = NPTL_MATCH(CTL_TAG, 0, MPI_ANY_SOURCE); - mes[i].ignore_bits = NPTL_MATCH_IGNORE; - - ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[i], - PTL_PRIORITY_LIST, (void *)(uint64_t)i, &me_handles[i]); + overflow_me.start = recvbufs + (i * BUFSIZE); + ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &overflow_me, + PTL_OVERFLOW_LIST, (void *)(size_t)i, &me_handles[i]); MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret)); } @@ -81,6 +83,8 @@ int MPID_nem_ptl_nm_finalize(void) MPID_nem_ptl_strerror(ret)); } + MPIU_Free(recvbufs); + fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_NM_FINALIZE); return mpi_errno; @@ -383,19 +387,17 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e) case PTL_EVENT_PUT: { int ret; - const uint64_t buf_idx = (uint64_t) e->user_ptr; const size_t packet_sz = e->mlength; MPIDI_VC_t *vc; MPID_nem_ptl_vc_area * vc_ptl; ptl_size_t remaining = NPTL_HEADER_GET_LENGTH(e->hdr_data); - - MPIU_Assert(e->start == &recvbufs[buf_idx * PTL_MAX_EAGER]); + ptl_me_t search_me; MPIDI_PG_Get_vc(MPIDI_Process.my_pg, NPTL_MATCH_GET_RANK(e->match_bits), &vc); vc_ptl = VC_PTL(vc); if (remaining == 0) { - mpi_errno = MPID_nem_handle_pkt(vc, &recvbufs[buf_idx * PTL_MAX_EAGER], packet_sz); + mpi_errno = MPID_nem_handle_pkt(vc, e->start, packet_sz); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } @@ -411,7 +413,7 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e) REQ_PTL(req)->bytes_put = packet_sz + remaining; TMPBUF(req) = MPIU_Malloc(REQ_PTL(req)->bytes_put); MPIU_Assert(TMPBUF(req) != NULL); - MPIU_Memcpy(TMPBUF(req), &recvbufs[buf_idx * PTL_MAX_EAGER], packet_sz); + MPIU_Memcpy(TMPBUF(req), e->start, packet_sz); req->ch.vc = vc; @@ -434,11 +436,13 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e) } } - /* Repost the recv buffer */ - ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[buf_idx], - PTL_PRIORITY_LIST, e->user_ptr /* buf_idx */, &me_handles[buf_idx]); - MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", - "**ptlmeappend %s", MPID_nem_ptl_strerror(ret)); + /* FIXME: this search/delete not be necessary if we set PTL_ME_UNEXPECTED_HDR_DISABLE + on the overflow list entries. However, doing so leads to PTL_IN_USE errors + during finalize in rare cases, even though all messages are handled. */ + memcpy(&search_me, &overflow_me, sizeof(ptl_me_t)); + search_me.options |= PTL_ME_USE_ONCE; + /* find and delete the header for this message in the overflow list */ + PtlMESearch(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &search_me, PTL_SEARCH_DELETE, NULL); } break; @@ -483,6 +487,23 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e) } break; + case PTL_EVENT_AUTO_FREE: + { + size_t buf_idx = (size_t)e->user_ptr; + int ret; + + overflow_me.start = recvbufs + (buf_idx * BUFSIZE); + + ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &overflow_me, + PTL_OVERFLOW_LIST, e->user_ptr, &me_handles[buf_idx]); + MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", + MPID_nem_ptl_strerror(ret)); + } + + case PTL_EVENT_AUTO_UNLINK: + case PTL_EVENT_PUT_OVERFLOW: + break; + default: MPIU_Error_printf("Received unexpected event type: %d %s\n", e->type, MPID_nem_ptl_strevent(e)); MPIU_ERR_INTERNALANDJUMP(mpi_errno, "Unexpected event type"); diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c index 3e22556..195791d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c @@ -132,6 +132,8 @@ int MPID_nem_ptl_poll(int is_blocking_poll) /* MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_POLL); */ while (1) { + int ctl_event = FALSE; + /* check EQs for events */ ret = MPID_nem_ptl_rptl_eqget(MPIDI_nem_ptl_eq, &event); MPIU_ERR_CHKANDJUMP(ret == PTL_EQ_DROPPED, mpi_errno, MPI_ERR_OTHER, "**eqdropped"); @@ -146,6 +148,8 @@ int MPID_nem_ptl_poll(int is_blocking_poll) if (ret == PTL_EQ_EMPTY) { ret = MPID_nem_ptl_rptl_eqget(MPIDI_nem_ptl_origin_eq, &event); MPIU_ERR_CHKANDJUMP(ret == PTL_EQ_DROPPED, mpi_errno, MPI_ERR_OTHER, "**eqdropped"); + } else { + ctl_event = TRUE; } /* all EQs are empty */ @@ -159,16 +163,17 @@ int MPID_nem_ptl_poll(int is_blocking_poll) MPID_nem_ptl_strlist(event.ptl_list), event.user_ptr, event.hdr_data, event.mlength, event.rlength)); MPIU_ERR_CHKANDJUMP2(event.ni_fail_type != PTL_NI_OK && event.ni_fail_type != PTL_NI_NO_MATCH, mpi_errno, MPI_ERR_OTHER, "**ptlni_fail", "**ptlni_fail %s %s", MPID_nem_ptl_strevent(&event), MPID_nem_ptl_strnifail(event.ni_fail_type)); + /* special case for events on the control portal */ + if (ctl_event) { + mpi_errno = MPID_nem_ptl_nm_ctl_event_handler(&event); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + continue; + } + switch (event.type) { case PTL_EVENT_PUT: if (event.ptl_list == PTL_OVERFLOW_LIST) break; - if (event.pt_index == MPIDI_nem_ptl_control_pt) { - mpi_errno = MPID_nem_ptl_nm_ctl_event_handler(&event); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - break; - } case PTL_EVENT_PUT_OVERFLOW: case PTL_EVENT_GET: case PTL_EVENT_SEND: ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/netmod/portals4/ptl_nm.c | 79 ++++++++++++------- .../channels/nemesis/netmod/portals4/ptl_poll.c | 17 +++-- 2 files changed, 61 insertions(+), 35 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-52-g3ccad54
by noreply＠mpich.org 07 Apr '15

07 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3ccad54bd508c2a48525f4204a1b1622c8c0bca9 (commit) via f4b253ca6b352e70a507ac585fa2c85965bdd530 (commit) via fe576505335b4d93e8c0c66c943f40f73fa31d90 (commit) via dc563e5a027ee7e6fb93fcff42880584963295fa (commit) via 3093fe702697dc53d69dff5134492211d6cb2098 (commit) via 4ab1144b469ffe8e3d0c393b9daa9e704aa31d80 (commit) via a6b35edc1a218d747f71e0b73aff0ffb17ad18ad (commit) via ec8ce7a9365930b1fc1a3bdb711dc20eac049b18 (commit) via ae5e72186ae56d097124b6abfdc4bc906c8052a4 (commit) via 211e8986c621369fa503ee99194069b5dbfd0f45 (commit) via 0bdc7e7411b24707aa67708f1779a2cac104cbb4 (commit) from 32e24c1db7163dfd7f13f8bba6c9321cc53c6972 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3ccad54bd508c2a48525f4204a1b1622c… commit 3ccad54bd508c2a48525f4204a1b1622c8c0bca9 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Tue Mar 31 14:50:52 2015 +0900 Add copyright in netmod-IB Also change individual author to organization names. Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c index cb047f0..b9b560e 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c @@ -1,7 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2012 NEC Corporation - * Author: Masamichi Takagi + * (C) 2014 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index ad1481b..21c080f 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -1,8 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * (C) 2012 Oct 14 Yutaka Ishikawa, ishikawa(a)is.s.u-tokyo.ac.jp + * (C) 2012 University of Tokyo + * (C) 2014-2015 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h index 0d6af7e..ebce06c 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h @@ -1,8 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * (C) 2012 Oct 14 Yutaka Ishikawa, ishikawa(a)is.s.u-tokyo.ac.jp + * (C) 2012 University of Tokyo + * (C) 2014 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 25c334a..faa3079 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -1,7 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2012 NEC Corporation - * Author: Masamichi Takagi + * (C) 2014-2015 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c index 804a066..9010a6b 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c @@ -1,10 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2006 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - * * (C) 2012 NEC Corporation - * Author: Masamichi Takagi + * (C) 2014-2015 RIKEN AICS + * + * See COPYRIGHT in top-level directory. */ #include "ib_impl.h" diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 95f72e0..1235fdf 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -1,7 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2013 NEC Corporation - * Author: Masamichi Takagi + * (C) 2014-2015 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c index fe08357..e9a21c5 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c @@ -1,3 +1,10 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2014-2015 RIKEN AICS + * + * See COPYRIGHT in top-level directory. + */ + #define _GNU_SOURCE 1 #include <stdio.h> #include <stdlib.h> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 1678d37..d3cab00 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -1,7 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2012 NEC Corporation - * Author: Masamichi Takagi + * (C) 2014 RIKEN AICS + * (C) 2014-2015 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c index c9dca68..78457eb 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c @@ -1,9 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* + * (C) 2001-2009 University of Tokyo * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * (C) 2012 Oct 10 Min Si - * (C) 2001-2009 Yutaka Ishikawa + * (C) 2012 University of Tokyo + * (C) 2014-2015 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index 5978bd9..701cc19 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -1,7 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2012 NEC Corporation - * Author: Masamichi Takagi + * (C) 2014 RIKEN AICS + * (C) 2014-2015 RIKEN AICS + * * See COPYRIGHT in top-level directory. */ http://git.mpich.org/mpich.git/commitdiff/f4b253ca6b352e70a507ac585fa2c8596… commit f4b253ca6b352e70a507ac585fa2c85965bdd530 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Tue Mar 31 14:38:16 2015 +0900 Fix operation of long messages in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 72cffc9..25c334a 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -722,9 +722,12 @@ typedef struct { void *addr; uint32_t rkey; uint8_t tail; /* last word of payload */ - int len; + long len; MPI_Request sender_req_id; /* request id of sender side */ + MPI_Request receiver_req_id; /* request id of sender side */ uint32_t max_msg_sz; /* max message size */ + int seg_seq_num; + int seg_num; } MPID_nem_ib_rma_lmt_cookie_t; typedef enum MPID_nem_ib_pkt_subtype { @@ -927,13 +930,13 @@ int pkt_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * bu } \ } while (0) -#define MPID_nem_ib_lmt_send_RTS(vc, _req_id, _addr, _rkey, _seg_seq_num) do { \ +#define MPID_nem_ib_lmt_send_RTS(_subtype, vc, _req_id, _addr, _rkey, _seg_seq_num) do { \ MPID_PKT_DECL_CAST(_upkt, MPID_nem_ib_pkt_lmt_rts_t, _rts_pkt); \ MPID_Request *_rts_req; \ \ MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending rndv RTS segment packet"); \ MPIDI_Pkt_init(_rts_pkt, MPIDI_NEM_PKT_NETMOD); \ - _rts_pkt->subtype = MPIDI_NEM_IB_PKT_LMT_RTS;\ + _rts_pkt->subtype = _subtype;\ _rts_pkt->req_id = _req_id; \ _rts_pkt->addr = _addr; \ _rts_pkt->rkey = _rkey; \ @@ -962,6 +965,7 @@ int pkt_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * bu MPIDI_Pkt_init(_done_pkt, MPIDI_NEM_PKT_NETMOD); \ _done_pkt->subtype = MPIDI_NEM_IB_PKT_RMA_LMT_GET_DONE;\ _done_pkt->req_id = (rreq)->ch.lmt_req_id; \ + _done_pkt->receiver_req_id = (rreq)->handle; \ /* embed SR occupancy information */ \ _done_pkt->seq_num_tail = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ \ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 0f6f1a6..95f72e0 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -186,7 +186,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3 MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &r_max_msg_sz, sizeof(uint32_t)); - divide = (max_msg_sz + r_max_msg_sz - 1) / r_max_msg_sz; + divide = (len + r_max_msg_sz - 1) / r_max_msg_sz; write_pos = write_to_buf; posted_num = 0; @@ -196,7 +196,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3 for (i = 0; i < divide; i++) { if (i == divide - 1) - data_sz = max_msg_sz - i * r_max_msg_sz; + data_sz = len - i * r_max_msg_sz; else data_sz = r_max_msg_sz; @@ -206,8 +206,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3 else last = MPID_NEM_IB_LMT_SEGMENT_LAST; /* last part of this segment */ - /* last data may be smaller than initiator's max_msg_sz */ - if (rest_data_sz < max_msg_sz) + if (rest_data_sz < r_max_msg_sz) data_sz = rest_data_sz; } diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 608e4be..1678d37 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -446,6 +446,9 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); MPIDI_CH3U_Request_complete(req); } + else if (req_wrap->mf == MPID_NEM_IB_LMT_SEGMENT_LAST) { + MPID_nem_ib_lmt_send_PKT_LMT_DONE(req->ch.vc, req); + } /* decrement the number of entries in IB command queue */ vc_ib->ibcom->ncom -= 1; @@ -1434,35 +1437,69 @@ int MPID_nem_ib_PktHandler_rma_lmt_rts(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); int mpi_errno = MPI_SUCCESS; MPID_Request *req = NULL; - MPIDI_CH3_Pkt_t *pkt_hdr = (MPIDI_CH3_Pkt_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t)); - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = - (MPID_nem_ib_rma_lmt_cookie_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t) + - sizeof(MPIDI_CH3_Pkt_t)); + MPID_nem_ib_pkt_lmt_rts_t *rts_pkt = (MPID_nem_ib_pkt_lmt_rts_t *) pkt; void *write_to_buf; + void *addr; + uint32_t rkey; + long length; + int last; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_RMA_LMT_RTS); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_RMA_LMT_RTS); - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); /* decrement only in drain_scq ? */ + if (rts_pkt->seg_seq_num == 1) { + // receive a packet for first segment + MPIDI_CH3_Pkt_t *pkt_hdr = (MPIDI_CH3_Pkt_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t)); + MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = + (MPID_nem_ib_rma_lmt_cookie_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t) + + sizeof(MPIDI_CH3_Pkt_t)); + + req = MPID_Request_create(); + MPIU_Object_set_ref(req, 1); /* decrement only in drain_scq ? */ + + req->ch.lmt_data_sz = s_cookie_buf->len; + req->ch.lmt_req_id = s_cookie_buf->sender_req_id; + + REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; + + REQ_FIELD(req, lmt_pack_buf) = + MPIU_Malloc(sizeof(MPIDI_CH3_Pkt_t) + (size_t) req->ch.lmt_data_sz); + MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER, + "**outofmemory"); + + memcpy(REQ_FIELD(req, lmt_pack_buf), pkt_hdr, sizeof(MPIDI_CH3_Pkt_t)); + REQ_FIELD(req, seg_num) = s_cookie_buf->seg_num; /* store number of segments */ - req->ch.lmt_data_sz = s_cookie_buf->len; - req->ch.lmt_req_id = s_cookie_buf->sender_req_id; + addr = s_cookie_buf->addr; + rkey = s_cookie_buf->rkey; - REQ_FIELD(req, lmt_pack_buf) = - MPIU_Malloc(sizeof(MPIDI_CH3_Pkt_t) + (size_t) req->ch.lmt_data_sz); - MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER, "**outofmemory"); + REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail; + } + else { + MPID_Request_get_ptr(rts_pkt->req_id, req); + addr = rts_pkt->addr; + rkey = rts_pkt->rkey; + } - memcpy(REQ_FIELD(req, lmt_pack_buf), pkt_hdr, sizeof(MPIDI_CH3_Pkt_t)); + if (rts_pkt->seg_seq_num == REQ_FIELD(req, seg_num)) { + last = 1; + length = + req->ch.lmt_data_sz - (long) (rts_pkt->seg_seq_num - 1) * REQ_FIELD(req, max_msg_sz); + } + else { + last = 0; + length = REQ_FIELD(req, max_msg_sz); + } /* RDMA READ buffer address */ - write_to_buf = (void *) ((char *) REQ_FIELD(req, lmt_pack_buf) + sizeof(MPIDI_CH3_Pkt_t)); + write_to_buf = + (void *) ((char *) REQ_FIELD(req, lmt_pack_buf) + sizeof(MPIDI_CH3_Pkt_t) + + (rts_pkt->seg_seq_num - 1) * REQ_FIELD(req, max_msg_sz)); /* stash vc for ib_poll */ req->ch.vc = vc; - REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail; MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_SEND); // Set dummy type for ib_drain_scq /* try to issue RDMA-read command */ @@ -1472,9 +1509,8 @@ int MPID_nem_ib_PktHandler_rma_lmt_rts(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { MPIDI_Request_set_msg_type(req, MPIDI_REQUEST_RNDV_MSG); mpi_errno = - MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, - s_cookie_buf->len, write_to_buf, - s_cookie_buf->max_msg_sz, 1); + MPID_nem_ib_lmt_start_recv_core(req, addr, rkey, length, write_to_buf, + REQ_FIELD(req, max_msg_sz), last); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1487,12 +1523,11 @@ int MPID_nem_ib_PktHandler_rma_lmt_rts(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY); /* make raddr, (sz is in rreq->ch.lmt_data_sz), rkey, (user_buf is in req->dev.user_buf) survive enqueue, free cookie, dequeue */ - REQ_FIELD(req, lmt_raddr) = s_cookie_buf->addr; - REQ_FIELD(req, lmt_rkey) = s_cookie_buf->rkey; + REQ_FIELD(req, lmt_raddr) = addr; + REQ_FIELD(req, lmt_rkey) = rkey; REQ_FIELD(req, lmt_write_to_buf) = write_to_buf; - REQ_FIELD(req, lmt_szsend) = s_cookie_buf->len; - REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; - REQ_FIELD(req, last) = 1; /* not support segmentation */ + REQ_FIELD(req, lmt_szsend) = length; + REQ_FIELD(req, last) = last; /* not support segmentation */ /* set for send_progress */ MPIDI_Request_set_msg_type(req, MPIDI_REQUEST_RNDV_MSG); @@ -1501,9 +1536,15 @@ int MPID_nem_ib_PktHandler_rma_lmt_rts(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req); } - /* prefix + header + data */ - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_rma_lmt_cookie_t); + if (rts_pkt->seg_seq_num == 1) { + /* prefix + header + data */ + *buflen = + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_CH3_Pkt_t) + + sizeof(MPID_nem_ib_rma_lmt_cookie_t); + } + else { + *buflen = sizeof(MPIDI_CH3_Pkt_t); + } *rreqp = NULL; fn_exit: @@ -1593,8 +1634,8 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, #else void *_addr = addr; #endif - MPID_nem_ib_lmt_send_RTS(vc, done_pkt->receiver_req_id, _addr, mr->rkey, - next_seg_seq_num); + MPID_nem_ib_lmt_send_RTS(MPIDI_NEM_IB_PKT_LMT_RTS, vc, done_pkt->receiver_req_id, _addr, + mr->rkey, next_seg_seq_num); } break; default: @@ -1819,32 +1860,62 @@ int MPID_nem_ib_pkt_rma_lmt_getdone(MPIDI_VC_t * vc, /* decrement reference counter of mr_cache_entry */ MPID_nem_ib_com_reg_mr_release(REQ_FIELD(req, lmt_mr_cache)); - req_type = MPIDI_Request_get_type(req); - /* free memory area for cookie */ - if (!req->ch.s_cookie) { - dprintf("lmt_done_send,enter,req->ch.s_cookie is zero"); - } - MPIU_Free(req->ch.s_cookie); + if (REQ_FIELD(req, seg_seq_num) == REQ_FIELD(req, seg_num)) { + req_type = MPIDI_Request_get_type(req); + /* free memory area for cookie */ + if (!req->ch.s_cookie) { + dprintf("lmt_done_send,enter,req->ch.s_cookie is zero"); + } + MPIU_Free(req->ch.s_cookie); - if ((req_type == 0 && !req->comm) || (req_type == MPIDI_REQUEST_TYPE_GET_RESP)) { - if ((*req->cc_ptr == 1) && req->dev.datatype_ptr && (req->dev.segment_size > 0) && - REQ_FIELD(req, lmt_pack_buf)) { - MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); + if ((req_type == 0 && !req->comm) || (req_type == MPIDI_REQUEST_TYPE_GET_RESP)) { + if ((*req->cc_ptr == 1) && req->dev.datatype_ptr && (req->dev.segment_size > 0) && + REQ_FIELD(req, lmt_pack_buf)) { + MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); + } } - } - int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); - reqFn = req->dev.OnDataAvail; + int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); + reqFn = req->dev.OnDataAvail; - if (*req->cc_ptr == 1 && - (reqFn == MPIDI_CH3_ReqHandler_ReqOpsComplete - || reqFn == MPIDI_CH3_ReqHandler_GetSendComplete)) { - MPIDI_VC_t *_vc = req->ch.vc; - int complete = 0; - mpi_errno = reqFn(_vc, req, &complete); + if (*req->cc_ptr == 1 && + (reqFn == MPIDI_CH3_ReqHandler_ReqOpsComplete + || reqFn == MPIDI_CH3_ReqHandler_GetSendComplete)) { + MPIDI_VC_t *_vc = req->ch.vc; + int complete = 0; + mpi_errno = reqFn(_vc, req, &complete); + } + else { + MPIDI_CH3U_Request_complete(req); + } } else { - MPIDI_CH3U_Request_complete(req); + REQ_FIELD(req, seg_seq_num) += 1; /* next segment number */ + int next_seg_seq_num = REQ_FIELD(req, seg_seq_num); + + uint32_t length; + if (next_seg_seq_num == REQ_FIELD(req, seg_num)) + length = REQ_FIELD(req, data_sz) - (long) (next_seg_seq_num - 1) * REQ_FIELD(req, max_msg_sz); //length of last segment + else + length = REQ_FIELD(req, max_msg_sz); + + void *addr = + (void *) ((char *) REQ_FIELD(req, buf.from) + + (long) (next_seg_seq_num - 1) * REQ_FIELD(req, max_msg_sz)); + struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = + MPID_nem_ib_com_reg_mr_fetch(addr, length, 0, MPID_NEM_IB_COM_REG_MR_GLOBAL); + MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); + struct ibv_mr *mr = mr_cache->mr; + /* store new cache entry */ + REQ_FIELD(req, lmt_mr_cache) = (void *) mr_cache; + +#ifdef HAVE_LIBDCFA + void *_addr = mr->host_addr; +#else + void *_addr = addr; +#endif + MPID_nem_ib_lmt_send_RTS(MPIDI_NEM_IB_PKT_RMA_LMT_RTS, vc, done_pkt->receiver_req_id, _addr, + mr->rkey, next_seg_seq_num); } *rreqp = NULL; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c index d97ed5d..c9dca68 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c @@ -223,7 +223,7 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, num_global_cache--; /* end loop if the total length released exceeds the requested */ - if (dereg_total > len_aligned) { + if (dereg_total >= len_aligned) { dprintf("ib_com_reg_mr_fetch,dereg=%llu,len=%ld\n", dereg_total, len_aligned); /* re-registraion */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index 39d6325..5978bd9 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -29,11 +29,11 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo int mpi_errno = MPI_SUCCESS; int ibcom_errno; MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPID_nem_ib_pkt_prefix_t pkt_netmod; + MPID_nem_ib_pkt_lmt_rts_t pkt_rts; void *prefix; int sz_prefix; void *s_data; - int s_data_sz; + long s_data_sz; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ISENDCONTIG_CORE); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_ISENDCONTIG_CORE); @@ -66,9 +66,9 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) + sizeof(MPIDI_CH3_Pkt_t) + data_sz > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { - pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; + pkt_rts.type = MPIDI_NEM_PKT_NETMOD; - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; + pkt_rts.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; void *write_from_buf = data; @@ -83,9 +83,19 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); + + int post_num = (data_sz + (long) max_msg_sz - 1) / (long) max_msg_sz; + long length; + if (post_num > 1) { + length = max_msg_sz; + } + else { + length = data_sz; + } + /* put IB rkey */ struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = - MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0, + MPID_nem_ib_com_reg_mr_fetch(write_from_buf, length, 0, MPID_NEM_IB_COM_REG_MR_GLOBAL); MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); struct ibv_mr *mr = mr_cache->mr; @@ -99,9 +109,19 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo s_cookie_buf->len = data_sz; s_cookie_buf->sender_req_id = sreq->handle; s_cookie_buf->max_msg_sz = max_msg_sz; + s_cookie_buf->seg_seq_num = 1; + s_cookie_buf->seg_num = post_num; + + pkt_rts.seg_seq_num = 1; + + REQ_FIELD(sreq, buf.from) = write_from_buf; + REQ_FIELD(sreq, data_sz) = data_sz; + REQ_FIELD(sreq, seg_seq_num) = 1; // only send 1st-segment, even if there are some segments. + REQ_FIELD(sreq, seg_num) = post_num; + REQ_FIELD(sreq, max_msg_sz) = max_msg_sz; /* set for ib_com_isend */ - prefix = (void *) &pkt_netmod; + prefix = (void *) &pkt_rts; sz_prefix = sizeof(MPIDI_CH3_Pkt_t); s_data = (void *) s_cookie_buf; s_data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t); @@ -465,8 +485,8 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *prefix; int prefix_sz; void *data; - int data_sz; - MPID_nem_ib_pkt_prefix_t pkt_netmod; + long data_sz; + MPID_nem_ib_pkt_lmt_rts_t pkt_rts; prefix = NULL; prefix_sz = 0; @@ -495,9 +515,9 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) + sizeof(MPIDI_CH3_Pkt_t) + data_sz > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { - pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; + pkt_rts.type = MPIDI_NEM_PKT_NETMOD; - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; + pkt_rts.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; void *write_from_buf = REQ_FIELD(sreq, lmt_pack_buf); @@ -512,9 +532,19 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); + + int post_num = (data_sz + (long) max_msg_sz - 1) / (long) max_msg_sz; + long length; + if (post_num > 1) { + length = max_msg_sz; + } + else { + length = data_sz; + } + /* put IB rkey */ struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = - MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0, + MPID_nem_ib_com_reg_mr_fetch(write_from_buf, length, 0, MPID_NEM_IB_COM_REG_MR_GLOBAL); MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); struct ibv_mr *mr = mr_cache->mr; @@ -528,9 +558,19 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, s_cookie_buf->len = data_sz; s_cookie_buf->sender_req_id = sreq->handle; s_cookie_buf->max_msg_sz = max_msg_sz; + s_cookie_buf->seg_seq_num = 1; + s_cookie_buf->seg_num = post_num; + + pkt_rts.seg_seq_num = 1; + + REQ_FIELD(sreq, buf.from) = write_from_buf; + REQ_FIELD(sreq, data_sz) = data_sz; + REQ_FIELD(sreq, seg_seq_num) = 1; // only send 1st-segment, even if there are some segments. + REQ_FIELD(sreq, seg_num) = post_num; + REQ_FIELD(sreq, max_msg_sz) = max_msg_sz; /* set for ib_com_isend */ - prefix = (void *) &pkt_netmod; + prefix = (void *) &pkt_rts; prefix_sz = sizeof(MPIDI_CH3_Pkt_t); data = (void *) s_cookie_buf; data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t); http://git.mpich.org/mpich.git/commitdiff/fe576505335b4d93e8c0c66c943f40f73… commit fe576505335b4d93e8c0c66c943f40f73fa31d90 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Mon Mar 30 17:25:44 2015 +0900 Fix some build warnings in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index 9064232..ad1481b 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -128,7 +128,7 @@ static int MPID_nem_ib_rdmawr_to_init(uint64_t sz) void *MPID_nem_ib_rdmawr_to_alloc(int nslots) { dprintf("rdmawr_to_alloc,nslots=%d\n", nslots); - void *start; + void *start = NULL; int i; for (i = 0; i < nslots; i++) { //dprintf("MPID_nem_ib_rdmawr_to_alloc,free_list=%p\n", MPID_nem_ib_rdmawr_to_alloc_free_list); @@ -505,7 +505,6 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) MPID_nem_ib_com_t *conp; struct ibv_qp_init_attr qp_init_attr; struct ibv_sge *sge; - int mr_flags; int i; dprintf("MPID_nem_ib_com_open,port=%d,flag=%08x\n", ib_port, open_flag); @@ -725,7 +724,6 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) memset(conp->icom_mem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); conp->icom_msize = (int *) MPIU_Malloc(sizeof(int *) * MPID_NEM_IB_COM_NBUF_RDMA); memset(conp->icom_msize, 0, sizeof(int *) * MPID_NEM_IB_COM_NBUF_RDMA); - mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; /* RDMA-write-to local memory area */ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO] = MPID_NEM_IB_COM_RDMABUF_SZ; @@ -783,7 +781,6 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) memset(conp->icom_mem, 0, sizeof(void *) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); conp->icom_msize = (int *) MPIU_Malloc(sizeof(int) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); memset(conp->icom_msize, 0, sizeof(int) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); - mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; /* RDMA-write-from local memory area */ conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_FROM] = MPID_NEM_IB_COM_SCRATCH_PAD_FROM_SZ; @@ -824,7 +821,6 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) memset(conp->icom_mem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_UD); conp->icom_msize = (int *) MPIU_Malloc(sizeof(int *) * MPID_NEM_IB_COM_NBUF_UD); memset(conp->icom_msize, 0, sizeof(int *) * MPID_NEM_IB_COM_NBUF_UD); - mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; /* UD-write-from local memory area */ MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, @@ -845,7 +841,7 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM], 0, MPID_NEM_IB_COM_REG_MR_STICKY); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + printf("ibv_reg_mr failed\n")); /* UD-write-to local memory area */ /* addr to addr+39 are not filled, addr+40 to addr+length-1 are filled with payload */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h index ac6fe0d..0d6af7e 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h @@ -772,7 +772,7 @@ static inline void *MPID_nem_ib_rdmawr_from_alloc(uint32_t _sz) static inline void MPID_nem_ib_rdmawr_from_free(const void *p, uint32_t _sz) { int clz; - uint32_t sz; + uint32_t sz _UNUSED_; assert(_sz <= (1ULL << 31)); MPID_NEM_IB_RDMAWR_FROM_ALLOC_PREPROCESS_SZ; void *q = MPID_NEM_IB_RDMAWR_FROM_ALLOC_ARENA_START(p); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 2acb65e..72cffc9 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -23,6 +23,12 @@ */ #define MPID_NEM_IB_ONDEMAND +#ifdef __GNUC__ +#define _UNUSED_ __attribute__ ((__unused__)) +#else +#define _UNUSED_ +#endif + typedef struct { union ibv_gid gid; uint16_t lid; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 1f51d95..0f6f1a6 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -271,7 +271,7 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP { int mpi_errno = MPI_SUCCESS; int dt_contig; - MPIDI_msg_sz_t data_sz; + MPIDI_msg_sz_t data_sz _UNUSED_; MPID_Datatype *dt_ptr; MPI_Aint dt_true_lb; MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 705593f..608e4be 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -605,7 +605,7 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) int mpi_errno = MPI_SUCCESS; int ibcom_errno; - struct MPIDI_VC *vc; + struct MPIDI_VC *vc = NULL; MPID_nem_ib_vc_area *vc_ib; //int result; //struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; @@ -616,7 +616,7 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) //MPID_nem_ib_tsc_poll = MPID_nem_ib_rdtsc(); - uint16_t *remote_poll; + uint16_t *remote_poll = NULL; switch (ringbuf->type) { case MPID_NEM_IB_RINGBUF_EXCLUSIVE: remote_poll = &VC_FIELD(ringbuf->vc, ibcom->rsr_seq_num_poll); @@ -1152,8 +1152,8 @@ int MPID_nem_ib_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) #endif MPIDI_msg_sz_t data_sz; - int dt_contig; - MPI_Aint dt_true_lb; + int dt_contig _UNUSED_; + MPI_Aint dt_true_lb _UNUSED_; MPID_Datatype *dt_ptr; MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb); @@ -1839,9 +1839,9 @@ int MPID_nem_ib_pkt_rma_lmt_getdone(MPIDI_VC_t * vc, if (*req->cc_ptr == 1 && (reqFn == MPIDI_CH3_ReqHandler_ReqOpsComplete || reqFn == MPIDI_CH3_ReqHandler_GetSendComplete)) { - MPIDI_VC_t *vc = req->ch.vc; + MPIDI_VC_t *_vc = req->ch.vc; int complete = 0; - mpi_errno = reqFn(vc, req, &complete); + mpi_errno = reqFn(_vc, req, &complete); } else { MPIDI_CH3U_Request_complete(req); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index 2624310..39d6325 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -716,7 +716,7 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) int mpi_errno = MPI_SUCCESS; MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); MPID_Request *sreq, *prev_sreq; - int req_type, msg_type; + int msg_type; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_SEND_PROGRESS); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_SEND_PROGRESS); @@ -740,7 +740,6 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) if (sreq) { prev_sreq = NULL; do { - req_type = MPIDI_Request_get_type(sreq); msg_type = MPIDI_Request_get_msg_type(sreq); MPIDI_CH3_Pkt_t *ch3_hdr = (MPIDI_CH3_Pkt_t *) sreq->dev.iov[0].MPID_IOV_BUF; http://git.mpich.org/mpich.git/commitdiff/dc563e5a027ee7e6fb93fcff428805849… commit dc563e5a027ee7e6fb93fcff42880584963295fa Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Wed Mar 25 15:44:51 2015 +0900 Fix invalid access before connection established Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index 58aec8a..9064232 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -36,6 +36,7 @@ static int maxcon; static struct ibv_device **ib_devlist; static struct ibv_context *ib_ctx; struct ibv_context *MPID_nem_ib_ctx_export; /* for SC13 demo connector */ +static struct ibv_port_attr ib_pattr; static struct ibv_pd *ib_pd; struct ibv_pd *MPID_nem_ib_pd_export; /* for SC13 demo connector */ struct ibv_cq *MPID_nem_ib_rc_shared_scq; @@ -275,6 +276,7 @@ static int MPID_nem_ib_com_device_init() int dev_num; char *dev_name; int i; + int ib_port = 1; if (ib_initialized == 1) { dprintf("MPID_nem_ib_com_device_init,already initialized\n"); @@ -314,6 +316,14 @@ static int MPID_nem_ib_com_device_init() } else { ib_ctx = ibv_open_device(ib_devlist[i]); + + if (ib_ctx) { + /* get port attribute */ + if (ibv_query_port(ib_ctx, ib_port, &ib_pattr)) { + dprintf("ibv_query_port on port %d failed\n", ib_port); + goto err_exit; + } + } } dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_ctx_export=%p,ib_ctx=%p\n", MPID_nem_ib_ctx_export, ib_ctx); @@ -2282,6 +2292,30 @@ int MPID_nem_ib_com_get_info_conn(int condesc, int key, void *out, uint32_t out_ goto fn_exit; } +int MPID_nem_ib_com_get_info_pattr(int key, void *out, uint32_t out_len) +{ + int ibcom_errno = 0; + + switch (key) { + case MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ:{ +#ifdef HAVE_LIBDCFA + uint32_t max_msg_sz = 1073741824; /* ConnectX-3 */ + memcpy(out, &max_msg_sz, out_len); +#else + memcpy(out, &ib_pattr.max_msg_sz, out_len); +#endif + break; + } + default: + ibcom_errno = -1; + break; + } + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + int MPID_nem_ib_com_get_info_mr(int condesc, int memid, int key, void *out, int out_len) { int ibcom_errno = 0; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h index ae28f55..ac6fe0d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h @@ -559,6 +559,7 @@ extern int MPID_nem_ib_com_reg_mr(void *addr, long len, struct ibv_mr **mr, extern int MPID_nem_ib_com_dereg_mr(struct ibv_mr *mr); extern int MPID_nem_ib_com_get_info_conn(int condesc, int key, void *out, uint32_t out_len); +extern int MPID_nem_ib_com_get_info_pattr(int key, void *out, uint32_t out_len); extern int MPID_nem_ib_com_get_info_mr(int condesc, int memid, int key, void *out, int out_len); extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(int condesc, int *notify_rate); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 9f9a4e5..1f51d95 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -101,8 +101,16 @@ int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_p int post_num; uint32_t max_msg_sz; MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, - &max_msg_sz, sizeof(uint32_t)); + + if (vc_ib->connection_state == MPID_NEM_IB_CM_ESTABLISHED) { + MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, + &max_msg_sz, sizeof(uint32_t)); + } + else { + /* If connection is not established, get max_msg_sz from the global value. */ + MPID_nem_ib_com_get_info_pattr(MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &max_msg_sz, + sizeof(uint32_t)); + } /* Type of max_msg_sz is uint32_t. */ post_num = (data_sz + (long) max_msg_sz - 1) / (long) max_msg_sz; http://git.mpich.org/mpich.git/commitdiff/3093fe702697dc53d69dff5134492211d… commit 3093fe702697dc53d69dff5134492211d6cb2098 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Thu Mar 19 14:21:44 2015 +0900 Fix sending size of non-contiguous data in IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index a185452..2624310 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -476,8 +476,8 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, last = sreq->dev.segment_size; /* segment_size is byte offset */ if (last > 0) { - REQ_FIELD(sreq, lmt_pack_buf) = - MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first)); + data_sz = sreq->dev.segment_size - sreq->dev.segment_first; + REQ_FIELD(sreq, lmt_pack_buf) = MPIU_Malloc((size_t) data_sz); MPIU_ERR_CHKANDJUMP(!REQ_FIELD(sreq, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER, "**outofmemory"); MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, @@ -486,14 +486,14 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, } else { REQ_FIELD(sreq, lmt_pack_buf) = NULL; + data_sz = 0; } data = (void *) REQ_FIELD(sreq, lmt_pack_buf); - data_sz = last; /* If request length is too long, create LMT packet */ if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) - + sizeof(MPIDI_CH3_Pkt_t) + sreq->dev.segment_size - sreq->dev.segment_first + + sizeof(MPIDI_CH3_Pkt_t) + data_sz > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; @@ -510,10 +510,11 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, sreq->ch.s_cookie = s_cookie_buf; - s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + last - sizeof(uint8_t))); + s_cookie_buf->tail = + *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); /* put IB rkey */ struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = - MPID_nem_ib_com_reg_mr_fetch(write_from_buf, last, 0, + MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0, MPID_NEM_IB_COM_REG_MR_GLOBAL); MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); struct ibv_mr *mr = mr_cache->mr; @@ -524,7 +525,7 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, s_cookie_buf->addr = write_from_buf; #endif s_cookie_buf->rkey = mr->rkey; - s_cookie_buf->len = last; + s_cookie_buf->len = data_sz; s_cookie_buf->sender_req_id = sreq->handle; s_cookie_buf->max_msg_sz = max_msg_sz; http://git.mpich.org/mpich.git/commitdiff/4ab1144b469ffe8e3d0c393b9daa9e704… commit 4ab1144b469ffe8e3d0c393b9daa9e704aa31d80 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Thu Mar 19 13:06:19 2015 +0900 Fix strict-aliasing rules break in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 2069aeb..2acb65e 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -55,8 +55,12 @@ typedef struct { /* macro for secret area in vc */ #define VC_CH(vc) ((MPIDI_CH3I_VC *)&(vc)->ch) -#define VC_IB(vc) ((MPID_nem_ib_vc_area *)VC_CH((vc))->netmod_area.padding) -#define VC_FIELD(vcp, field) (((MPID_nem_ib_vc_area *)VC_CH(((vcp)))->netmod_area.padding)->field) +static inline MPID_nem_ib_vc_area *VC_IB(MPIDI_VC_t * vc) +{ + return (MPID_nem_ib_vc_area *) vc->ch.netmod_area.padding; +} + +#define VC_FIELD(vcp, field) VC_IB(vcp)->field /* The req provides a generic buffer in which network modules can store private fields This removes all dependencies from the req structure @@ -88,8 +92,12 @@ typedef struct { } MPID_nem_ib_req_area; /* macro for secret area in req */ -#define REQ_IB(req) ((MPID_nem_ib_req_area *)(&(req)->ch.netmod_area.padding)) -#define REQ_FIELD(reqp, field) (((MPID_nem_ib_req_area *)((reqp)->ch.netmod_area.padding))->field) +static inline MPID_nem_ib_req_area *REQ_IB(MPID_Request * req) +{ + return (MPID_nem_ib_req_area *) req->ch.netmod_area.padding; +} + +#define REQ_FIELD(reqp, field) (REQ_IB(reqp)->field) /* see src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h */ /* sreq is never enqueued into posted-queue nor unexpected-queue, so we can reuse sreq->dev.next */ http://git.mpich.org/mpich.git/commitdiff/a6b35edc1a218d747f71e0b73aff0ffb1… commit a6b35edc1a218d747f71e0b73aff0ffb17ad18ad Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Wed Mar 18 17:56:30 2015 +0900 Fix memory leak in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 92590e1..705593f 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -153,9 +153,7 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) if (req_type == MPIDI_REQUEST_TYPE_SEND && req->comm) { /* exclude control messages by requiring MPIDI_REQUEST_TYPE_SEND * exclude eager-short by requiring req->comm != 0 */ - int is_contig; - MPID_Datatype_is_contig(req->dev.datatype, &is_contig); - if (!is_contig && REQ_FIELD(req, lmt_pack_buf)) { + if (REQ_FIELD(req, lmt_pack_buf)) { dprintf("drain_scq,eager-send,non-contiguous,free lmt_pack_buf=%p\n", REQ_FIELD(req, lmt_pack_buf)); MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index dfcef02..a185452 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -111,6 +111,8 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone } + REQ_FIELD(sreq, lmt_pack_buf) = NULL; + /* packet handlers including MPIDI_CH3_PktHandler_EagerSend and MPID_nem_handle_pkt assume this */ hdr_sz = sizeof(MPIDI_CH3_Pkt_t); @@ -482,6 +484,9 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, (char *) REQ_FIELD(sreq, lmt_pack_buf)); MPIU_Assert(last == sreq->dev.segment_size); } + else { + REQ_FIELD(sreq, lmt_pack_buf) = NULL; + } data = (void *) REQ_FIELD(sreq, lmt_pack_buf); data_sz = last; http://git.mpich.org/mpich.git/commitdiff/ec8ce7a9365930b1fc1a3bdb711dc20ea… commit ec8ce7a9365930b1fc1a3bdb711dc20eac049b18 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Wed Mar 18 15:38:31 2015 +0900 Fix allocation size and release size in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 4ad5d07..2069aeb 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -985,10 +985,16 @@ static inline void *MPID_nem_ib_stmalloc(size_t _sz) sz >>= 1; } while (sz > 0); if (i < 12) { - return MPIU_Malloc(sz); + return MPIU_Malloc(_sz); } if (i > 30) { - return mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + void *addr = mmap(0, _sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (addr == (void *) -1) { + return NULL; + } + else { + return addr; + } } int ndx = i - 12; void *slot; @@ -1008,12 +1014,13 @@ static inline void *MPID_nem_ib_stmalloc(size_t _sz) return slot; } -static inline void MPID_nem_ib_stfree(void *ptr, size_t sz) +static inline void MPID_nem_ib_stfree(void *ptr, size_t _sz) { if (MPID_nem_ib_myrank == 1) { //printf("stfree,%p,%08x\n", ptr, (int)sz); } int i = 0; + size_t sz = _sz; do { i++; sz >>= 1; @@ -1023,7 +1030,7 @@ static inline void MPID_nem_ib_stfree(void *ptr, size_t sz) goto fn_exit; } if (i > 30) { - munmap(ptr, sz); + munmap(ptr, _sz); goto fn_exit; } int ndx = i - 12; http://git.mpich.org/mpich.git/commitdiff/ae5e72186ae56d097124b6abfdc4bc906… commit ae5e72186ae56d097124b6abfdc4bc906c8052a4 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Mon Mar 16 13:17:25 2015 +0900 Fix fail to build with new libc Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c index d2b679e..fe08357 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_malloc.c @@ -29,7 +29,10 @@ void *ib_malloc_hook(size_t size, const void *caller); void ib_free_hook(void *addr, const void *caller); void *ib_realloc_hook(void *addr, size_t size, const void *caller); -void (*__malloc_initialize_hook) (void) = _local_malloc_initialize_hook; +#ifndef __MALLOC_HOOK_VOLATILE +#define __MALLOC_HOOK_VOLATILE +#endif +void (*__MALLOC_HOOK_VOLATILE __malloc_initialize_hook) (void) = _local_malloc_initialize_hook; static pthread_mutex_t mutex; static int __initialized_malloc = 0; http://git.mpich.org/mpich.git/commitdiff/211e8986c621369fa503ee99194069b5d… commit 211e8986c621369fa503ee99194069b5dbfd0f45 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Thu Mar 12 14:37:09 2015 +0900 Code cleanup in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index d943bb7..58aec8a 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -1900,8 +1900,8 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset /* rkey is defined in MPID_nem_ib_com_reg_mr_connect */ dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr. - rdma.remote_addr); + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma. + remote_addr); #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]); @@ -1965,8 +1965,8 @@ int MPID_nem_ib_com_get_scratch_pad(int condesc, /* rkey is defined in MPID_nem_ib_com_reg_mr_connect */ dprintf("MPID_nem_ib_com_get_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_GET].wr. - rdma.remote_addr); + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_GET].wr.rdma. + remote_addr); #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_GET]); @@ -2032,8 +2032,8 @@ int MPID_nem_ib_com_cas_scratch_pad(int condesc, conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS].wr.atomic.swap = swap; dprintf("MPID_nem_ib_com_cas_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS].wr. - rdma.remote_addr); + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS].wr.rdma. + remote_addr); #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS]); @@ -2456,7 +2456,7 @@ char *MPID_nem_ib_com_strerror(int err) goto fn_exit; } else { - r = (char *)strerror_tbl[-err]; + r = (char *) strerror_tbl[-err]; } fn_exit: return r; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 889d3ba..4ad5d07 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -484,7 +484,7 @@ static inline int MPID_nem_ib_cbf_hash1(uint64_t addr) (((addr >> (MPID_nem_ib_cbf_lognslot * 0)) & (MPID_nem_ib_cbf_nslot - 1)) ^ ((addr >> (MPID_nem_ib_cbf_lognslot * 3)) & (MPID_nem_ib_cbf_nslot - 1)) ^ (((addr >> (MPID_nem_ib_cbf_lognslot * 6)) & (MPID_nem_ib_cbf_nslot - 1)) - + 1)) & (MPID_nem_ib_cbf_nslot - 1); + + 1)) & (MPID_nem_ib_cbf_nslot - 1); } static inline int MPID_nem_ib_cbf_hash2(uint64_t addr) @@ -494,7 +494,7 @@ static inline int MPID_nem_ib_cbf_hash2(uint64_t addr) (((addr >> (MPID_nem_ib_cbf_lognslot * 1)) & (MPID_nem_ib_cbf_nslot - 1)) ^ ((addr >> (MPID_nem_ib_cbf_lognslot * 4)) & (MPID_nem_ib_cbf_nslot - 1)) ^ (((addr >> (MPID_nem_ib_cbf_lognslot * 7)) & (MPID_nem_ib_cbf_nslot - 1)) - + 1)) & (MPID_nem_ib_cbf_nslot - 1); + + 1)) & (MPID_nem_ib_cbf_nslot - 1); } static inline int MPID_nem_ib_cbf_hash3(uint64_t addr) @@ -504,7 +504,7 @@ static inline int MPID_nem_ib_cbf_hash3(uint64_t addr) (((addr >> (MPID_nem_ib_cbf_lognslot * 2)) & (MPID_nem_ib_cbf_nslot - 1)) ^ ((addr >> (MPID_nem_ib_cbf_lognslot * 5)) & (MPID_nem_ib_cbf_nslot - 1)) ^ (((addr >> (MPID_nem_ib_cbf_lognslot * 8)) & (MPID_nem_ib_cbf_nslot - 1)) - + 2)) & (MPID_nem_ib_cbf_nslot - 1); + + 2)) & (MPID_nem_ib_cbf_nslot - 1); } diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c index 62d46eb..804a066 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c @@ -494,7 +494,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m for (i = 0; i < MPID_nem_ib_nranks; i++) { if (i != MPID_nem_ib_myrank) { for (j = 0; j < MPID_NEM_IB_COM_MAX_RQ_CAPACITY; j++) { - MPID_nem_ib_com_scratch_pad_recv(MPID_nem_ib_scratch_pad_fds[i], sizeof(MPID_nem_ib_cm_notify_send_t)); + MPID_nem_ib_com_scratch_pad_recv(MPID_nem_ib_scratch_pad_fds[i], + sizeof(MPID_nem_ib_cm_notify_send_t)); } } } @@ -855,11 +856,9 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); int ntrial = 0; - volatile MPID_nem_ib_com_qp_state_t *rstate = - (MPID_nem_ib_com_qp_state_t *) ((uint8_t *) - MPID_nem_ib_com_scratch_pad->icom_mem - [MPID_NEM_IB_COM_SCRATCH_PAD_TO] + - vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t)); + volatile MPID_nem_ib_com_qp_state_t *rstate = (MPID_nem_ib_com_qp_state_t *) + ((uint8_t *) MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + + vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t)); dprintf("ib_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate)); while (rstate->state != MPID_NEM_IB_COM_QP_STATE_RTR) { __asm__ __volatile__("pause;":::"memory"); @@ -1005,8 +1004,8 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) dprintf ("vc_terminate,before,%d->%d,diff-rsr=%d,l diff-lsr=%d,sendq_empty=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff16(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom-> - rsr_seq_num_tail_last_sent), + vc_ib-> + ibcom->rsr_seq_num_tail_last_sent), MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 716ac03..9f9a4e5 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -323,8 +323,8 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP length = s_cookie_buf->max_msg_sz; } - REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; /* store initiator's max_msg_sz */ - REQ_FIELD(req, seg_num) = s_cookie_buf->seg_num; /* store number of segments */ + REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; /* store initiator's max_msg_sz */ + REQ_FIELD(req, seg_num) = s_cookie_buf->seg_num; /* store number of segments */ /* try to issue RDMA-read command */ int slack = 1; /* slack for control packet bringing sequence number */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 0e64960..92590e1 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -1270,20 +1270,14 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) MPID_nem_ib_ringbuf_headtail_t *headtail = (MPID_nem_ib_ringbuf_headtail_t *) ((uint8_t *) MPID_nem_ib_scratch_pad + MPID_NEM_IB_RINGBUF_OFF_HEAD); - uint16_t index_tail = - vc_ib->ibcom->remote_ringbuf->type == - MPID_NEM_IB_RINGBUF_EXCLUSIVE ? ((uint16_t) (vc_ib->ibcom->rsr_seq_num_tail + 1) % - vc_ib->ibcom-> - remote_ringbuf->nslot) : ((uint16_t) (headtail->tail + - 1) % - vc_ib->ibcom->remote_ringbuf-> - nslot); + uint16_t index_tail = vc_ib->ibcom->remote_ringbuf->type == MPID_NEM_IB_RINGBUF_EXCLUSIVE ? + ((uint16_t) (vc_ib->ibcom->rsr_seq_num_tail + 1) % vc_ib->ibcom->remote_ringbuf-> nslot) : + ((uint16_t) (headtail->tail + 1) % vc_ib->ibcom->remote_ringbuf->nslot); dprintf("released,index_tail=%d\n", index_tail); dprintf("released,%016lx\n", vc_ib->ibcom->remote_ringbuf->remote_released[index_tail / 64]); if (1 || (index_tail & 7) || MPID_nem_ib_diff16(index_slot, index_tail) >= vc_ib->ibcom->remote_ringbuf->nslot - 8) { /* avoid wrap-around */ while (1) { - if (((vc_ib->ibcom->remote_ringbuf-> - remote_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) { + if (((vc_ib->ibcom-> remote_ringbuf->remote_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) { if (vc_ib->ibcom->remote_ringbuf->type == MPID_NEM_IB_RINGBUF_EXCLUSIVE) { vc_ib->ibcom->rsr_seq_num_tail += 1; dprintf("exclusive ringbuf,remote_tail,incremented to %d\n", @@ -1414,7 +1408,7 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, * see MPID_nem_ib_iSendContig */ //ch3_pkt->type = MPIDI_CH3_PKT_EAGER_SEND; - dprintf("ib_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); + dprintf("ib_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_EagerSend(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); dprintf("ib_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen); @@ -1660,8 +1654,7 @@ int MPID_nem_ib_pkt_RTS_handler(MPIDI_VC_t * vc, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { mpi_errno = MPID_nem_ib_lmt_start_recv_core(req, rts_pkt->addr, rts_pkt->rkey, length, - write_to_buf, REQ_FIELD(req, max_msg_sz), - last); + write_to_buf, REQ_FIELD(req, max_msg_sz), last); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1955,8 +1948,8 @@ int MPID_nem_ib_cm_drain_scq() MPID_nem_ib_cm_ringbuf_tail) < MPID_NEM_IB_CM_NSEG) { MPID_nem_ib_cm_cmd_syn_t *cmd = - (MPID_nem_ib_cm_cmd_syn_t *) shadow_cm->req->ibcom-> - icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; + (MPID_nem_ib_cm_cmd_syn_t *) shadow_cm->req-> + ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; MPID_NEM_IB_CM_COMPOSE_SYN(cmd, shadow_cm->req); cmd->responder_ringbuf_index = shadow_cm->req->responder_ringbuf_index = @@ -2481,8 +2474,8 @@ int MPID_nem_ib_cm_poll_syn() MPID_nem_ib_cm_ringbuf_tail) < MPID_NEM_IB_CM_NSEG) { MPID_nem_ib_cm_cmd_synack_t *cmd = - (MPID_nem_ib_cm_cmd_synack_t *) req->ibcom-> - icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; + (MPID_nem_ib_cm_cmd_synack_t *) req-> + ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; if (is_synack) { dprintf("cm_poll_syn,sending synack,%d->%d[%d],connection_tx=%d\n", MPID_nem_ib_myrank, syn->initiator_rank, req->ringbuf_index, @@ -2722,8 +2715,8 @@ int MPID_nem_ib_cm_poll() dprintf("cm_poll,sending ack1,req=%p,ringbuf_index=%d\n", req, req->ringbuf_index); MPID_nem_ib_cm_cmd_ack1_t *cmd = - (MPID_nem_ib_cm_cmd_ack1_t *) req->ibcom-> - icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; + (MPID_nem_ib_cm_cmd_ack1_t *) req-> + ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; MPID_NEM_IB_CM_COMPOSE_ACK1(cmd, req, synack->responder_req); dprintf ("cm_poll,composing ack1,cmd->responder_req=%p,cmd->rmem=%p,rkey=%08x,ringbuf_nslot=%d,remote_vc=%p\n", @@ -2863,8 +2856,8 @@ int MPID_nem_ib_cm_poll() req, req->ringbuf_index, req->initiator_rank, req->ibcom->outstanding_connection_tx); MPID_nem_ib_cm_cmd_ack2_t *cmd = - (MPID_nem_ib_cm_cmd_ack2_t *) req->ibcom-> - icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; + (MPID_nem_ib_cm_cmd_ack2_t *) req-> + ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; MPID_NEM_IB_CM_COMPOSE_ACK2(cmd, ack1->initiator_req); MPID_nem_ib_cm_cmd_shadow_t *shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c index bd8b5f5..d97ed5d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c @@ -132,8 +132,7 @@ static inline void __lru_queue_display() int i = 0; for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { dprintf("---- hash %d\n", i); - for (p = - (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) + for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next; p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i]; p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) { @@ -357,8 +356,7 @@ int MPID_nem_ib_com_register_cache_release() } for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { - for (p = - (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) + for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next; p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];) { diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index 8981409..dfcef02 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -62,53 +62,54 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo s_data = data; s_data_sz = data_sz; - /* If request length is too long, create LMT packet */ - if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) - + sizeof(MPIDI_CH3_Pkt_t) + data_sz - > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { - pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; - - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; - - void *write_from_buf = data; - - uint32_t max_msg_sz; - MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, - &max_msg_sz, sizeof(uint32_t)); - - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t)); - - sreq->ch.s_cookie = s_cookie_buf; - - s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); - /* put IB rkey */ - struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = - MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0, - MPID_NEM_IB_COM_REG_MR_GLOBAL); - MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_com_reg_mr_fetch"); - struct ibv_mr *mr = mr_cache->mr; - REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache; + /* If request length is too long, create LMT packet */ + if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) + + sizeof(MPIDI_CH3_Pkt_t) + data_sz + > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { + pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; + + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; + + void *write_from_buf = data; + + uint32_t max_msg_sz; + MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, + &max_msg_sz, sizeof(uint32_t)); + + MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = + (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t)); + + sreq->ch.s_cookie = s_cookie_buf; + + s_cookie_buf->tail = + *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); + /* put IB rkey */ + struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = + MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0, + MPID_NEM_IB_COM_REG_MR_GLOBAL); + MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); + struct ibv_mr *mr = mr_cache->mr; + REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache; #ifdef HAVE_LIBDCFA - s_cookie_buf->addr = (void *) mr->host_addr; + s_cookie_buf->addr = (void *) mr->host_addr; #else - s_cookie_buf->addr = write_from_buf; + s_cookie_buf->addr = write_from_buf; #endif - s_cookie_buf->rkey = mr->rkey; - s_cookie_buf->len = data_sz; - s_cookie_buf->sender_req_id = sreq->handle; - s_cookie_buf->max_msg_sz = max_msg_sz; - - /* set for ib_com_isend */ - prefix = (void *)&pkt_netmod; - sz_prefix = sizeof(MPIDI_CH3_Pkt_t); - s_data = (void *)s_cookie_buf; - s_data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t); - - /* Release Request, when sender receives DONE packet. */ - int incomplete; - MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone - } + s_cookie_buf->rkey = mr->rkey; + s_cookie_buf->len = data_sz; + s_cookie_buf->sender_req_id = sreq->handle; + s_cookie_buf->max_msg_sz = max_msg_sz; + + /* set for ib_com_isend */ + prefix = (void *) &pkt_netmod; + sz_prefix = sizeof(MPIDI_CH3_Pkt_t); + s_data = (void *) s_cookie_buf; + s_data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t); + + /* Release Request, when sender receives DONE packet. */ + int incomplete; + MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone + } /* packet handlers including MPIDI_CH3_PktHandler_EagerSend and MPID_nem_handle_pkt assume this */ hdr_sz = sizeof(MPIDI_CH3_Pkt_t); @@ -473,7 +474,8 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, last = sreq->dev.segment_size; /* segment_size is byte offset */ if (last > 0) { - REQ_FIELD(sreq, lmt_pack_buf) = MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first)); + REQ_FIELD(sreq, lmt_pack_buf) = + MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first)); MPIU_ERR_CHKANDJUMP(!REQ_FIELD(sreq, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER, "**outofmemory"); MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, @@ -481,56 +483,56 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, MPIU_Assert(last == sreq->dev.segment_size); } - data = (void *)REQ_FIELD(sreq, lmt_pack_buf); + data = (void *) REQ_FIELD(sreq, lmt_pack_buf); data_sz = last; - /* If request length is too long, create LMT packet */ - if ( MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) - + sizeof(MPIDI_CH3_Pkt_t) + sreq->dev.segment_size - sreq->dev.segment_first - > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { - pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; + /* If request length is too long, create LMT packet */ + if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) + + sizeof(MPIDI_CH3_Pkt_t) + sreq->dev.segment_size - sreq->dev.segment_first + > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { + pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; - void *write_from_buf = REQ_FIELD(sreq, lmt_pack_buf); + void *write_from_buf = REQ_FIELD(sreq, lmt_pack_buf); - uint32_t max_msg_sz; - MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, - &max_msg_sz, sizeof(uint32_t)); + uint32_t max_msg_sz; + MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, + &max_msg_sz, sizeof(uint32_t)); - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t)); + MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = + (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t)); - sreq->ch.s_cookie = s_cookie_buf; + sreq->ch.s_cookie = s_cookie_buf; - s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + last - sizeof(uint8_t))); - /* put IB rkey */ - struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = - MPID_nem_ib_com_reg_mr_fetch(write_from_buf, last, 0, - MPID_NEM_IB_COM_REG_MR_GLOBAL); - MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_com_reg_mr_fetch"); - struct ibv_mr *mr = mr_cache->mr; - REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache; + s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + last - sizeof(uint8_t))); + /* put IB rkey */ + struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache = + MPID_nem_ib_com_reg_mr_fetch(write_from_buf, last, 0, + MPID_NEM_IB_COM_REG_MR_GLOBAL); + MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); + struct ibv_mr *mr = mr_cache->mr; + REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache; #ifdef HAVE_LIBDCFA - s_cookie_buf->addr = (void *) mr->host_addr; + s_cookie_buf->addr = (void *) mr->host_addr; #else - s_cookie_buf->addr = write_from_buf; + s_cookie_buf->addr = write_from_buf; #endif - s_cookie_buf->rkey = mr->rkey; - s_cookie_buf->len = last; - s_cookie_buf->sender_req_id = sreq->handle; - s_cookie_buf->max_msg_sz = max_msg_sz; - - /* set for ib_com_isend */ - prefix = (void *)&pkt_netmod; - prefix_sz = sizeof(MPIDI_CH3_Pkt_t); - data = (void *)s_cookie_buf; - data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t); - - /* Release Request, when sender receives DONE packet. */ - int incomplete; - MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone - } + s_cookie_buf->rkey = mr->rkey; + s_cookie_buf->len = last; + s_cookie_buf->sender_req_id = sreq->handle; + s_cookie_buf->max_msg_sz = max_msg_sz; + + /* set for ib_com_isend */ + prefix = (void *) &pkt_netmod; + prefix_sz = sizeof(MPIDI_CH3_Pkt_t); + data = (void *) s_cookie_buf; + data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t); + + /* Release Request, when sender receives DONE packet. */ + int incomplete; + MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone + } /* packet handlers assume this */ hdr_sz = sizeof(MPIDI_CH3_Pkt_t); @@ -873,8 +875,8 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) } case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:{ MPID_nem_ib_pkt_reply_seq_num_t *_pkt = - (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.iov[0]. - MPID_IOV_BUF; + (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev. + iov[0].MPID_IOV_BUF; dprintf ("send_progress,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); @@ -1016,8 +1018,7 @@ int MPID_nem_ib_cm_progress() ("cm_progress,retry CAS,responder_rank=%d,req=%p,decided=%ld,vt=%ld,backoff=%ld\n", sreq->responder_rank, sreq, sreq->retry_decided, MPID_nem_ib_progress_engine_vt, sreq->retry_backoff); - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1030,13 +1031,13 @@ int MPID_nem_ib_cm_progress() ("cm_progress,retry CAS_RELEASE,responder_rank=%d,req=%p,decided=%ld,vt=%ld,backoff=%ld\n", sreq->responder_rank, sreq, sreq->retry_decided, MPID_nem_ib_progress_engine_vt, sreq->retry_backoff); - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; mpi_errno = MPID_nem_ib_cm_cas_release_core(sreq->responder_rank, shadow); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_cas_release_core"); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_cm_cas_release_core"); break; case MPID_NEM_IB_CM_SYN: if (is_conn_established(sreq->responder_rank)) { @@ -1061,8 +1062,7 @@ int MPID_nem_ib_cm_progress() ((MPID_nem_ib_cm_cmd_syn_t *) & sreq->cmd)->initiator_rank = MPID_nem_ib_myrank; MPID_nem_ib_cm_ringbuf_head++; - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1080,8 +1080,7 @@ int MPID_nem_ib_cm_progress() ((MPID_nem_ib_cm_cmd_syn_t *) & sreq->cmd)->initiator_rank = MPID_nem_ib_myrank; - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1103,8 +1102,7 @@ int MPID_nem_ib_cm_progress() MPID_nem_ib_cm_ringbuf_head; sreq->initiator_ringbuf_index = MPID_nem_ib_cm_ringbuf_head; MPID_nem_ib_cm_ringbuf_head++; - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1118,8 +1116,7 @@ int MPID_nem_ib_cm_progress() "**MPID_nem_ib_cm_send_core"); break; case MPID_NEM_IB_CM_ACK1: - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1133,8 +1130,7 @@ int MPID_nem_ib_cm_progress() "**MPID_nem_ib_cm_send_core"); break; case MPID_NEM_IB_CM_ACK2: - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1149,8 +1145,7 @@ int MPID_nem_ib_cm_progress() break; case MPID_NEM_IB_CM_ALREADY_ESTABLISHED: case MPID_NEM_IB_CM_RESPONDER_IS_CONNECTING: - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) + shadow = (MPID_nem_ib_cm_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1236,13 +1231,8 @@ int MPID_nem_ib_cm_cas_core(int rank, MPID_nem_ib_cm_cmd_shadow_t * shadow) (unsigned long) conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + 0); /* Compare-and-swap rank to acquire communication manager port */ - ibcom_errno = - MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], - (uint64_t) shadow, - 0, - MPID_NEM_IB_CM_RELEASED, - MPID_nem_ib_myrank/*rank*/, /*debug*/ - &shadow->buf_from, &shadow->buf_from_sz); + ibcom_errno = MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) shadow, 0, MPID_NEM_IB_CM_RELEASED, MPID_nem_ib_myrank /*rank */ , /*debug */ + &shadow->buf_from, &shadow->buf_from_sz); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_cas_scratch_pad"); MPID_nem_ib_ncqe_scratch_pad += 1; @@ -1338,13 +1328,8 @@ int MPID_nem_ib_cm_cas_release_core(int rank, MPID_nem_ib_cm_cmd_shadow_t * shad (unsigned long) conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + 0); /* Compare-and-swap rank to acquire communication manager port */ - ibcom_errno = - MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], - (uint64_t) shadow, - 0, - MPID_nem_ib_myrank, - MPID_NEM_IB_CM_RELEASED/*rank*/, /*debug*/ - &shadow->buf_from, &shadow->buf_from_sz); + ibcom_errno = MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) shadow, 0, MPID_nem_ib_myrank, MPID_NEM_IB_CM_RELEASED /*rank */ , /*debug */ + &shadow->buf_from, &shadow->buf_from_sz); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_cas_scratch_pad"); MPID_nem_ib_ncqe_scratch_pad += 1; @@ -1531,11 +1516,13 @@ int MPID_nem_ib_cm_notify_progress(void) } ibcom_errno = MPID_nem_ib_cm_notify_send(sreq->pg_rank, sreq->my_rank); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_notify_send"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_cm_notify_send"); /* unlink sreq */ if (prev_sreq != NULL) { - MPID_nem_ib_cm_notify_sendq_next(prev_sreq) = MPID_nem_ib_cm_notify_sendq_next(sreq); + MPID_nem_ib_cm_notify_sendq_next(prev_sreq) = + MPID_nem_ib_cm_notify_sendq_next(sreq); } else { MPID_nem_ib_cm_notify_sendq_head(MPID_nem_ib_cm_notify_sendq) = @@ -1641,8 +1628,7 @@ int MPID_nem_ib_ringbuf_ask_fetch(MPIDI_VC_t * vc) * from the same process */ VC_FIELD(vc, ibcom->ask_guard) = 1; - MPID_nem_ib_ringbuf_cmd_shadow_t *shadow = - (MPID_nem_ib_ringbuf_cmd_shadow_t *) + MPID_nem_ib_ringbuf_cmd_shadow_t *shadow = (MPID_nem_ib_ringbuf_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_ringbuf_cmd_shadow_t)); shadow->type = req->state; shadow->req = req; @@ -1732,8 +1718,7 @@ int MPID_nem_ib_ringbuf_ask_cas(MPIDI_VC_t * vc, MPID_nem_ib_ringbuf_req_t * req dprintf("ask_cas,core\n"); req->state = MPID_NEM_IB_RINGBUF_ASK_CAS; - MPID_nem_ib_ringbuf_cmd_shadow_t *shadow = - (MPID_nem_ib_ringbuf_cmd_shadow_t *) + MPID_nem_ib_ringbuf_cmd_shadow_t *shadow = (MPID_nem_ib_ringbuf_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_ringbuf_cmd_shadow_t)); shadow->type = req->state; shadow->req = req; @@ -1799,8 +1784,7 @@ int MPID_nem_ib_ringbuf_progress() switch (sreq->state) { case MPID_NEM_IB_RINGBUF_ASK_CAS: dprintf("ringbuf_progress,ask_cas,req=%p\n", sreq); - shadow = - (MPID_nem_ib_ringbuf_cmd_shadow_t *) + shadow = (MPID_nem_ib_ringbuf_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_ringbuf_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; @@ -1829,8 +1813,7 @@ int MPID_nem_ib_ringbuf_progress() } dprintf("ringbuf_progress,ask_fetch,req=%p\n", sreq); VC_FIELD(sreq->vc, ibcom->ask_guard) = 1; - shadow = - (MPID_nem_ib_ringbuf_cmd_shadow_t *) + shadow = (MPID_nem_ib_ringbuf_cmd_shadow_t *) MPIU_Malloc(sizeof(MPID_nem_ib_ringbuf_cmd_shadow_t)); shadow->type = sreq->state; shadow->req = sreq; http://git.mpich.org/mpich.git/commitdiff/0bdc7e7411b24707aa67708f1779a2cac… commit 0bdc7e7411b24707aa67708f1779a2cac104cbb4 Author: Norio Yamaguchi <norio.yamaguchi(a)riken.jp> Date: Thu Mar 12 14:27:48 2015 +0900 Delete unused code in netmod-IB Signed-off-by: Huiwei Lu <huiweilu(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c index c34a392..cb047f0 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c @@ -24,22 +24,10 @@ int MPID_nem_ib_finalize(void) { int mpi_errno = MPI_SUCCESS; -#if 0 - int ibcom_errno; - int i; -#endif MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_FINALIZE); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_FINALIZE); -#if 0 - for (i = 0; i < MPID_nem_ib_nranks; i++) { - ibcom_errno = MPID_nem_ib_com_close(MPID_nem_ib_conns[i].fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_close"); - - } -#endif - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_FINALIZE); fn_exit: diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index 30bc222..d943bb7 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -221,7 +221,6 @@ static int modify_qp_to_rtr(struct ibv_qp *qp, uint32_t remote_qpn, uint16_t dli attr.ah_attr.port_num = ib_port; /* In dcfa gid is not set and for testing here it is also not set */ -#if 1 #ifdef HAVE_LIBDCFA /* DCFA doesn't use gid */ #else if (gid_idx >= 0) { @@ -234,7 +233,6 @@ static int modify_qp_to_rtr(struct ibv_qp *qp, uint32_t remote_qpn, uint16_t dli attr.ah_attr.grh.traffic_class = 0; } #endif -#endif flags = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; @@ -392,15 +390,7 @@ static int MPID_nem_ib_com_clean(MPID_nem_ib_com_t * conp) MPID_nem_ib_rc_shared_rcq = NULL; } -#if 0 /* It's not used */ - retval = - munmap(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(retval, -1, dprintf("munmap")); -#endif -#if 0 /* Don't free it because it's managed through VC_FILED(vc, ibcom->remote_ringbuf) */ - retval = munmap(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], MPID_NEM_IB_COM_RDMABUF_SZ); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(retval, -1, dprintf("munmap")); -#endif + MPIU_Free(conp->icom_mrlist); MPIU_Free(conp->icom_mem); MPIU_Free(conp->icom_msize); @@ -729,17 +719,7 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) /* RDMA-write-to local memory area */ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO] = MPID_NEM_IB_COM_RDMABUF_SZ; -#if 0 - int shmid = shmget(2, MPID_NEM_IB_COM_RDMABUF_SZ, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(shmid < 0, -1, perror("shmget")); - conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] = shmat(shmid, 0, 0); - if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] == (char *) -1) { - perror("Shared memory attach failure"); - shmctl(shmid, IPC_RMID, NULL); - ibcom_errno = -1; - goto fn_fail; - } -#else + /* ibv_reg_mr all memory area for all ring buffers * including shared and exclusive ones */ if (!MPID_nem_ib_rdmawr_to_alloc_start) { @@ -755,7 +735,6 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) //-1, 0); dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], MPID_NEM_IB_COM_RDMABUF_SZ); -#endif #ifdef HAVE_LIBDCFA dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, @@ -1131,14 +1110,6 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 0; conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].port_num = conp->icom_port; -#if 0 - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 1; - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.flow_label = 0; - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */ - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.hop_limit = 1; - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.traffic_class = 0; -#endif - /* SR (send request) template for MPID_NEM_IB_COM_UD_INITIATOR */ conp->icom_sr = (struct ibv_send_wr *) MPIU_Calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, @@ -1521,16 +1492,12 @@ int MPID_nem_ib_com_isend(int condesc, off_pow2_aligned, sz_pad, num_sge); conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].num_sge = num_sge; -#if 1 MPID_nem_ib_rc_send_request *wrap_wr_id = MPIU_Malloc(sizeof(MPID_nem_ib_rc_send_request)); wrap_wr_id->wr_id = wr_id; wrap_wr_id->mf = MPID_NEM_IB_LAST_PKT; wrap_wr_id->mr_cache = (void *) mr_cache; conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id = (uint64_t) wrap_wr_id; -#else - conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id = wr_id; -#endif conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.remote_addr = (uint64_t) conp->local_ringbuf_start + MPID_NEM_IB_COM_RDMABUF_SZSEG * ((uint16_t) (conp->sseq_num % conp->local_ringbuf_nslot)); @@ -1549,13 +1516,6 @@ int MPID_nem_ib_com_isend(int condesc, //dprintf("MPID_nem_ib_com_isend,condesc=%d,num_sge=%d,opcode=%08x,imm_data=%08x,wr_id=%016lx, raddr=%p, rkey=%08x\n", condesc, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].num_sge, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].opcode, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].imm_data, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.remote_addr, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey); - /* other commands can executed RDMA-rd command */ - /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ -#if 0 - if (conp->after_rdma_rd) { - conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags |= IBV_SEND_FENCE; - } -#endif #ifdef MPID_NEM_IB_ENABLE_INLINE if (sumsz <= conp->max_inline_data) { conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags |= IBV_SEND_INLINE; @@ -1582,12 +1542,6 @@ int MPID_nem_ib_com_isend(int condesc, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags &= ~IBV_SEND_INLINE; } #endif -#if 0 - if (conp->after_rdma_rd) { - conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags &= ~IBV_SEND_FENCE; - conp->after_rdma_rd = 0; - } -#endif conp->sseq_num += 1; conp->ncom += 1; @@ -1597,237 +1551,6 @@ int MPID_nem_ib_com_isend(int condesc, goto fn_exit; } -#if 0 -int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, - int sz_data) -{ - MPID_nem_ib_com_t *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - int sz_data_rem = sz_data; - int i; - struct ibv_mr *mr_data; - uint32_t sumsz = - sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr + sz_data + sizeof(MPID_nem_ib_netmod_trailer_t); - unsigned long tscs, tsce; - - dprintf("MPID_nem_ib_com_isend_chain,enter\n"); - MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_connected == 0, -1, - printf("MPID_nem_ib_com_isend_chain,icom_connected==0\n")); - - void *buf_from = - (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] + - MPID_NEM_IB_COM_RDMABUF_SZSEG * - ((uint16_t) (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG)); - - /* make a tail-magic position is in a fixed set */ - int off_pow2_aligned; - MPID_NEM_IB_OFF_POW2_ALIGNED(sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr + sz_data); - - /* let the last command icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAIN-1] which has IBV_WR_RDMA_WRITE_WITH_IMM */ - int s = - MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - (sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr + - off_pow2_aligned + - sizeof(MPID_nem_ib_netmod_trailer_t) + - MPID_NEM_IB_COM_INLINE_DATA - - 1) / MPID_NEM_IB_COM_INLINE_DATA; - MPID_NEM_IB_COM_ERR_CHKANDJUMP((sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr + - off_pow2_aligned) % 4 != 0, -1, - printf - ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(s < 0 || - s >= MPID_NEM_IB_COM_SMT_INLINE_NCHAIN, -1, - printf("MPID_nem_ib_com_isend_chain,s\n")); - dprintf("MPID_nem_ib_com_isend_chain,sz_hdr=%d,sz_data=%d,s=%d\n", sz_hdr, sz_data, s); - - for (i = s; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { - - //tscs = MPID_nem_ib_rdtsc(); - int sz_used = 0; /* how much of the payload of a IB packet is used? */ - int num_sge = 0; - if (i == s) { - MPID_nem_ib_netmod_hdr_t *netmod_hdr = (MPID_nem_ib_netmod_hdr_t *) buf_from; - MPID_NEM_IB_NETMOD_HDR_SZ_SET(netmod_hdr, sumsz); - memcpy((uint8_t *) buf_from + sizeof(MPID_nem_ib_netmod_hdr_t), hdr, sz_hdr); -#ifdef HAVE_LIBDCFA - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) buf_from; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) - conp->icom_mem - [MPID_NEM_IB_COM_RDMAWR_FROM]); -#else - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) buf_from; -#endif - buf_from = (uint8_t *) buf_from + sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = - sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr; - sz_used += sizeof(MPID_nem_ib_netmod_hdr_t) + sz_hdr; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; - num_sge += 1; - dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d\n", i, sz_used); - } - //tsce = MPID_nem_ib_rdtsc(); printf("0,%ld\n", tsce-tscs); - - //tscs = MPID_nem_ib_rdtsc(); - if (sz_data_rem > 0) { -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) data + sz_data - sz_data_rem; -#endif - int sz_data_red = - sz_used + sz_data_rem + sizeof(MPID_nem_ib_netmod_trailer_t) <= - MPID_NEM_IB_COM_INLINE_DATA ? sz_data_rem : sz_data_rem <= - MPID_NEM_IB_COM_INLINE_DATA - sz_used ? sz_data_rem : MPID_NEM_IB_COM_INLINE_DATA - - sz_used; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = - sz_data_red; - sz_used += sz_data_red; - sz_data_rem -= sz_data_red; - MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem < 0, -1, - printf("MPID_nem_ib_com_isend_chain,sz_data_rem\n")); - - if (i == s) { - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, - printf("MPID_nem_ib_com_isend_chain,sz_data==0\n")); - mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data, 0); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, - printf - ("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); - } -#ifdef HAVE_LIBDCFA - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) data + sz_data - sz_data_rem; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - mr_data->host_addr + ((uint64_t) data + sz_data - sz_data_rem - (uint64_t) data); -#endif - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - mr_data->lkey; - num_sge += 1; - dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used, - sz_data_rem); - } - else { /* netmod_trailer only packet is being generated */ - - } - //tsce = MPID_nem_ib_rdtsc(); printf("1,%ld\n", tsce-tscs); - - //tscs = MPID_nem_ib_rdtsc(); - if (i == MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) { /* append netmod_trailer */ - int sz_pad = off_pow2_aligned - sz_data; - MPID_nem_ib_netmod_trailer_t *netmod_trailer = - (MPID_nem_ib_netmod_trailer_t *) ((uint8_t *) buf_from + sz_pad); - netmod_trailer->tail_flag = MPID_NEM_IB_COM_MAGIC; -#ifdef HAVE_LIBDCFA - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) buf_from; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) - conp->icom_mem - [MPID_NEM_IB_COM_RDMAWR_FROM]); -#else - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) buf_from; -#endif - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = - sz_pad + sizeof(MPID_nem_ib_netmod_trailer_t); - sz_used += sz_pad + sizeof(MPID_nem_ib_netmod_trailer_t); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem != 0, -1, - printf("MPID_nem_ib_com_isend_chain, sz_data_rem\n")); - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; - num_sge += 1; - - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].imm_data = conp->sseq_num; - dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad, - sz_used, num_sge); - } - else if (MPID_NEM_IB_COM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */ - MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, - printf - ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); - int sz_pad = MPID_NEM_IB_COM_INLINE_DATA - sz_used; -#ifdef HAVE_LIBDCFA - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) buf_from; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) - conp->icom_mem - [MPID_NEM_IB_COM_RDMAWR_FROM]); -#else - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) buf_from; -#endif - buf_from = (uint8_t *) buf_from + sz_pad; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad; - sz_used += sz_pad; - MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1, - printf("MPID_nem_ib_com_isend_chain, sz_used\n")); - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; - num_sge += 1; - dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d\n", i, sz_pad, sz_used); - } - else { /* packet is full with data */ - MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1, - printf("MPID_nem_ib_com_isend_chain, sz_used\n")); - } - //tsce = MPID_nem_ib_rdtsc(); printf("2,%ld\n", tsce-tscs); - - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].num_sge = num_sge; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr_id = wr_id; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.remote_addr = - (uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] + - MPID_NEM_IB_COM_RDMABUF_SZSEG * - ((uint16_t) (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG)) + - MPID_NEM_IB_COM_INLINE_DATA * (i - s); - } -#if 0 - if (conp->after_rdma_rd) { - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s].send_flags |= IBV_SEND_FENCE; - } -#endif -#ifdef HAVE_LIBDCFA - ib_errno = - ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s]); -#else - ib_errno = - ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s], - &bad_wr); -#endif -#if 0 - if (i == 0 && conp->after_rdma_rd) { - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s].send_flags &= ~IBV_SEND_FENCE; - conp->after_rdma_rd = 0; - } -#endif -#ifdef HAVE_LIBDCFA - MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", - ib_errno)); -#else - MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf - ("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", - ib_errno, bad_wr)); -#endif - conp->ncom += (MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - s); - conp->sseq_num += 1; - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} -#endif - int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id) { @@ -1880,9 +1603,6 @@ int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remo #else /* prepare ibv_ah_attr */ conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].dlid = remote_lid; -#if 0 - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.dgid = *remote_gid; -#endif /* prepare ibv_ah */ struct ibv_ah *ah; @@ -1902,12 +1622,6 @@ int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remo conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].wr_id = wr_id; conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].imm_data = imm_data; -#if 0 - if (length <= qpinfo->max_inline_data) { - conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].send_flags |= IBV_SEND_INLINE; - } -#endif - #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR]); #else @@ -2010,11 +1724,6 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, long sz_data ib_errno, bad_wr)); #endif - /* other commands can be executed before RDMA-rd command */ - /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ -#if 0 - conp->after_rdma_rd = 1; -#endif conp->ncom += 1; fn_exit: @@ -2061,16 +1770,12 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat num_sge += 1; conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].num_sge = num_sge; -#if 1 MPID_nem_ib_rc_send_request *wrap_wr_id = MPIU_Malloc(sizeof(MPID_nem_ib_rc_send_request)); wrap_wr_id->wr_id = wr_id; wrap_wr_id->mf = MPID_NEM_IB_LAST_PKT; wrap_wr_id->mr_cache = (void *) mr_cache; conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr_id = (uint64_t) wrap_wr_id; -#else - conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr_id = wr_id; -#endif conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr.rdma.remote_addr = (uint64_t) raddr; conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr.rdma.rkey = rkey; @@ -2639,24 +2344,6 @@ int MPID_nem_ib_com_mem_rdmawr_from(int condesc, void **out) goto fn_exit; } -#if 0 -int MPID_nem_ib_com_mem_rdmawr_to(int condesc, int seq_num, void **out) -{ - MPID_nem_ib_com_t *conp; - int ibcom_errno = 0; - - MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = - (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + - MPID_NEM_IB_COM_RDMABUF_SZSEG * (seq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} -#endif - int MPID_nem_ib_com_mem_udwr_from(int condesc, void **out) { MPID_nem_ib_com_t *conp; @@ -2753,29 +2440,6 @@ int MPID_nem_ib_com_obtain_pointer(int condesc, MPID_nem_ib_com_t ** MPID_nem_ib goto fn_exit; } -#if 0 -static void MPID_nem_ib_comShow(int condesc) -{ - MPID_nem_ib_com_t *conp; - uint8_t *p; - int i; - - MPID_NEM_IB_RANGE_CHECK(condesc, conp); - fprintf(stdout, "qp_num = %d\n", conp->icom_qp->qp_num); -#ifdef HAVE_LIBDCFA - fprintf(stdout, "lid = %d\n", ib_ctx->lid); -#else - fprintf(stdout, "lid = %d\n", conp->icom_pattr.lid); -#endif - p = (uint8_t *) & conp->icom_gid; - fprintf(stdout, "gid = %02x", p[0]); - for (i = 1; i < 16; i++) { - fprintf(stdout, ":%02x", p[i]); - } - fprintf(stdout, "\n"); -} -#endif - static const char *strerror_tbl[] = { [0] = "zero", [1] = "one", diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index bfa0a3f..889d3ba 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -698,9 +698,6 @@ extern uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * typedef struct { void *addr; uint32_t rkey; -#if 0 /* moving to packet header */ - int seq_num_tail; /* notify RDMA-write-to buffer occupation */ -#endif uint8_t tail; /* last word of payload */ uint32_t max_msg_sz; /* max message size */ int seg_seq_num; @@ -718,9 +715,6 @@ typedef struct { typedef enum MPID_nem_ib_pkt_subtype { MPIDI_NEM_IB_PKT_EAGER_SEND, -#if 0 /* modification of mpid_nem_lmt.c is required */ - MPIDI_NEM_IB_PKT_LMT_RTS, -#endif MPIDI_NEM_IB_PKT_RMA_LMT_RTS, MPIDI_NEM_IB_PKT_PUT, MPIDI_NEM_IB_PKT_ACCUMULATE, @@ -791,28 +785,9 @@ typedef struct MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t { int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */); -#if 0 /* modification of mpid_nem_lmt.c is required */ -int MPID_nem_ib_pkt_RTS_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -#endif int MPID_nem_ib_PktHandler_rma_lmt_rts(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */); -#if 0 -int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -#endif int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c index b7adeab..62d46eb 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c @@ -499,12 +499,6 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m } } -#if 0 - MPIU_CHKPMEM_MALLOC(MPID_nem_ib_pollingset, MPIDI_VC_t **, - MPID_NEM_IB_MAX_POLLINGSET * sizeof(MPIDI_VC_t *), mpi_errno, - "connection table"); - memset(MPID_nem_ib_pollingset, 0, MPID_NEM_IB_MAX_POLLINGSET * sizeof(MPIDI_VC_t *)); -#endif #ifndef MPID_NEM_IB_ONDEMAND /* prepare eager-send QP */ for (i = 0; i < MPID_nem_ib_nranks; i++) { @@ -596,12 +590,6 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m } } - -#if 0 /* debug */ - for (i = 0; i < MPID_nem_ib_nranks; i++) { - dprintf("init,fd[%d]=%d\n", i, MPID_nem_ib_conns[i].fd); - } -#endif #else /* define(MPID_NEM_IB_ONDEMAND) */ /* We need to communicate with all other ranks in close sequence. */ MPID_nem_ib_conns_ref_count = MPID_nem_ib_nranks - MPID_nem_mem_region.num_local; @@ -821,13 +809,6 @@ int MPID_nem_ib_vc_onconnect(MPIDI_VC_t * vc) MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd, &VC_FIELD(vc, ibcom)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); -#if 0 - /* Insert into polling set */ - MPIU_ERR_CHKANDJUMP(MPID_nem_ib_npollingset + 1 > MPID_NEM_IB_MAX_POLLINGSET, mpi_errno, - MPI_ERR_OTHER, "**MPID_nem_ib_npollingset"); - MPID_nem_ib_pollingset[MPID_nem_ib_npollingset++] = vc; - //printf("vc_init,%d->%d,vc=%p,npollingset=%d\n", MPID_nem_ib_myrank, vc->pg_rank, vc, MPID_nem_ib_npollingset); -#endif fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_VC_ONCONNECT); return mpi_errno; @@ -903,29 +884,12 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) #endif MPIDI_CHANGE_VC_STATE(vc, ACTIVE); -#if 0 /* dead code */ - uint32_t max_msg_sz; - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[vc->pg_rank].fd, - MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &max_msg_sz, - sizeof(max_msg_sz)); -#endif VC_FIELD(vc, pending_sends) = 0; //MPIU_Assert(sizeof(MPID_nem_ib_netmod_hdr_t) == 8); /* assumption in ib_ibcom.h */ MPIU_Assert(sizeof(MPID_nem_ib_netmod_trailer_t) == 1); /* assumption in ib_ibcom.h */ uint32_t sz; -#if 0 - /* assumption in released(), must be power of two */ - sz = MPID_NEM_IB_COM_RDMABUF_NSEG; - while ((sz & 1) == 0) { - sz >>= 1; - } - sz >>= 1; - if (sz) { - MPIU_Assert(0); - } -#endif /* assumption in ib_poll.c, must be power of two */ for (sz = MPID_NEM_IB_COM_RDMABUF_SZSEG; sz > 0; sz >>= 1) { @@ -951,16 +915,7 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) vc_ch->pkt_handler = MPID_nem_ib_pkt_handler; vc_ch->num_pkt_handlers = MPIDI_NEM_IB_PKT_NUM_PKT_HANDLERS; MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_EAGER_SEND] = MPID_nem_ib_PktHandler_EagerSend; -#if 0 /* modification of mpid_nem_lmt.c is required */ - MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_LMT_RTS] = MPID_nem_ib_pkt_RTS_handler; -#endif MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_RMA_LMT_RTS] = MPID_nem_ib_PktHandler_rma_lmt_rts; -#if 0 - MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_PUT] = MPID_nem_ib_PktHandler_Put; - MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_GET] = MPID_nem_ib_PktHandler_Get; - MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_GET_RESP] = MPID_nem_ib_PktHandler_GetResp; - MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_ACCUMULATE] = MPID_nem_ib_PktHandler_Accumulate; -#endif MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_LMT_GET_DONE] = MPID_nem_ib_pkt_GET_DONE_handler; MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_LMT_RTS] = MPID_nem_ib_pkt_RTS_handler; MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REQ_SEQ_NUM] = MPID_nem_ib_PktHandler_req_seq_num; @@ -1055,21 +1010,6 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); - /* update remote RDMA-write-to buffer occupancy */ -#if 0 /* we can't send it when the other party has closed QP */ - while (MPID_nem_ib_diff16 - (vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent) > 0) { - MPID_nem_ib_send_reply_seq_num(vc); - } -#endif - - /* update local RDMA-write-to buffer occupancy */ -#if 0 - while (MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail) > 0) { - MPID_nem_ib_poll_eager(vc); - } -#endif - #ifdef MPID_NEM_IB_ONDEMAND MPID_nem_ib_cm_notify_send_req_t *req = MPIU_Malloc(sizeof(MPID_nem_ib_cm_notify_send_req_t)); req->ibcom = MPID_nem_ib_scratch_pad_ibcoms[vc->pg_rank]; @@ -1116,13 +1056,6 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); -#if 0 - if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { - usleep(1000); - MPID_nem_ib_drain_scq(0); - } -#endif - dprintf("init,middle2,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff16(vc_ib->ibcom->rsr_seq_num_tail, @@ -1134,15 +1067,6 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) usleep(1000); MPID_nem_ib_drain_scq(0); } -#if 0 - /* drain scq */ - while (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { - usleep(1000); - MPID_nem_ib_drain_scq(0); - //printf("%d\n", VC_FIELD(vc, pending_sends)); - //printf("%d\n", MPID_nem_ib_ncqe); - } -#endif dprintf("init,after ,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, @@ -1170,50 +1094,6 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) MPIU_ERR_POP(mpi_errno); } -#if 0 /* We move this code to the end of poll_eager. */ - /* Destroy VC QP */ - - /* Destroy ring-buffer */ - ibcom_errno = MPID_nem_ib_ringbuf_free(vc); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_ringbuf_free"); - - /* Check connection status stored in VC when on-demand connection is used */ - dprintf("vc_terminate,%d->%d,close\n", MPID_nem_ib_myrank, vc->pg_rank); - ibcom_errno = MPID_nem_ib_com_close(vc_ib->sc->fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_close"); - - /* Destroy array of scratch-pad QPs */ - MPIU_Assert(MPID_nem_ib_conns_ref_count > 0); - if (--MPID_nem_ib_conns_ref_count == 0) { - MPIU_Free(MPID_nem_ib_conns); - } - - /* TODO don't create them for shared memory vc */ - - /* Destroy scratch-pad */ - ibcom_errno = MPID_nem_ib_com_free(MPID_nem_ib_scratch_pad_fds[vc->pg_rank], -#ifdef MPID_NEM_IB_ONDEMAND - MPID_NEM_IB_CM_OFF_CMD + - MPID_NEM_IB_CM_NSEG * sizeof(MPID_nem_ib_cm_cmd_t) + - sizeof(MPID_nem_ib_ringbuf_headtail_t) -#else - MPID_nem_ib_nranks * sizeof(MPID_nem_ib_com_qp_state_t) -#endif -); - - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_free"); - - /* Destroy scratch-pad QP */ - ibcom_errno = MPID_nem_ib_com_close(MPID_nem_ib_scratch_pad_fds[vc->pg_rank]); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_close"); - - /* Destroy array of scratch-pad QPs */ - MPIU_Assert(MPID_nem_ib_scratch_pad_fds_ref_count > 0); - if (--MPID_nem_ib_scratch_pad_fds_ref_count == 0) { - MPIU_Free(MPID_nem_ib_scratch_pad_fds); - MPIU_Free(MPID_nem_ib_scratch_pad_ibcoms); - } -#endif dprintf("vc_terminate,exit\n"); fn_exit: diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 4941ccb..716ac03 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -30,9 +30,6 @@ int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_p MPIDI_msg_sz_t data_sz; MPID_Datatype *dt_ptr; MPI_Aint dt_true_lb; -#if 0 - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); -#endif MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_INITIATE_LMT); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_INITIATE_LMT); @@ -101,15 +98,6 @@ int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_p /* prepare magic */ //*((uint32_t*)(write_from_buf + data_sz - sizeof(tailmagic_t))) = MPID_NEM_IB_COM_MAGIC; -#if 0 /* moving to packet header */ /* embed RDMA-write-to buffer occupancy information */ - dprintf("lmt_initiate_lmt,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); - /* embed RDMA-write-to buffer occupancy information */ - s_cookie_buf->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; - - /* remember the last one sent */ - vc_ib->ibcom->rsr_seq_num_tail_last_sent = vc_ib->ibcom->rsr_seq_num_tail; -#endif - int post_num; uint32_t max_msg_sz; MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); @@ -367,14 +355,6 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req); } -#if 0 /* moving to packet header */ - /* extract embeded RDMA-write-to buffer occupancy information */ - dprintf("lmt_start_recv,old lsr_seq_num=%d,s_cookie_buf->seq_num_tail=%d\n", - vc_ib->ibcom->lsr_seq_num_tail, s_cookie_buf->seq_num_tail); - vc_ib->ibcom->lsr_seq_num_tail = s_cookie_buf->seq_num_tail; - //dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail); -#endif - #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); @@ -404,83 +384,6 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP goto fn_exit; } -#if 0 /* unused function */ -/* fall-back to lmt-get if end-flag of send-buf has the same value as the end-flag of recv-buf */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_lmt_switch_send -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) -{ - int mpi_errno = MPI_SUCCESS; - int dt_contig; - MPIDI_msg_sz_t data_sz; - MPID_Datatype *dt_ptr; - MPI_Aint dt_true_lb; - MPID_IOV r_cookie = req->ch.lmt_tmp_cookie; - MPID_nem_ib_lmt_cookie_t *r_cookie_buf = r_cookie.iov_base; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND); - - MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, - dt_true_lb); - - void *write_from_buf; - if (dt_contig) { - write_from_buf = req->dev.user_buf; - } - else { - /* see MPIDI_CH3_EagerNoncontigSend (in ch3u_eager.c) */ - req->dev.segment_ptr = MPID_Segment_alloc(); - MPIU_ERR_CHKANDJUMP((req->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, - "**outofmemory"); - - MPID_Segment_init(req->dev.user_buf, req->dev.user_count, req->dev.datatype, - req->dev.segment_ptr, 0); - req->dev.segment_first = 0; - req->dev.segment_size = data_sz; - - MPIDI_msg_sz_t last; - last = req->dev.segment_size; /* segment_size is byte offset */ - MPIU_Assert(last > 0); - - REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc(data_sz); - MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER, - "**outofmemory"); - - MPID_Segment_pack(req->dev.segment_ptr, req->dev.segment_first, &last, - (char *) (REQ_FIELD(req, lmt_pack_buf))); - MPIU_Assert(last == req->dev.segment_size); - - write_from_buf = REQ_FIELD(req, lmt_pack_buf); - } - - //assert(dt_true_lb == 0); - uint8_t *tailp = - (uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t)); -#if 0 - *is_end_flag_same = (r_cookie_buf->tail == *tailp) ? 1 : 0; -#else - REQ_FIELD(req, lmt_receiver_tail) = r_cookie_buf->tail; - REQ_FIELD(req, lmt_sender_tail) = *tailp; - dprintf("lmt_switch_send,tail on sender=%02x,tail onreceiver=%02x,req=%p\n", *tailp, - r_cookie_buf->tail, req); -#ifdef MPID_NEM_IB_DEBUG_LMT - uint8_t *tail_wordp = (uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint32_t) * 2); -#endif - dprintf("lmt_switch_send,tail on sender=%d\n", *tail_wordp); - fflush(stdout); -#endif - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif - /* when cookie is received in the middle of the lmt */ #undef FUNCNAME #define FUNCNAME MPID_nem_ib_lmt_handle_cookie @@ -531,9 +434,7 @@ int MPID_nem_ib_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req) MPID_Datatype_is_contig(req->dev.datatype, &is_contig); if (!is_contig && REQ_FIELD(req, lmt_pack_buf)) { dprintf("lmt_done_send,lmt-get,non-contiguous,free lmt_pack_buf\n"); -#if 1 /* debug, enable again later */ MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); -#endif } /* mark completion on sreq */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index fa2f3c9..0e64960 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -20,9 +20,6 @@ static int entered_drain_scq = 0; -#if 0 -#define MPID_NEM_IB_SEND_PROGRESS_POLLINGSET MPID_nem_ib_send_progress(vc); -#else #define MPID_NEM_IB_SEND_PROGRESS_POLLINGSET { \ do { \ int n; \ @@ -35,29 +32,13 @@ static int entered_drain_scq = 0; } \ } while (0); \ } -#if 0 - int n; \ - for(n = 0; n < MPID_nem_ib_npollingset; n++) { \ - MPIDI_VC_t *vc_n = MPID_nem_ib_pollingset[n]; \ - /*MPID_nem_ib_debug_current_vc_ib = vc_ib;*/ \ - MPID_nem_ib_send_progress(vc_n); \ - } \ -#endif -#endif -#if 1 #define MPID_NEM_IB_CHECK_AND_SEND_PROGRESS \ do { \ if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) { \ MPID_nem_ib_send_progress(vc); \ } \ } while (0) -#else -#define MPID_NEM_IB_CHECK_AND_SEND_PROGRESS \ - do { \ - MPID_NEM_IB_SEND_PROGRESS_POLLINGSET; \ - } while (0) -#endif #undef FUNCNAME #define FUNCNAME MPID_nem_ib_drain_scq @@ -90,12 +71,8 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) } #endif -#if 0 /*def HAVE_LIBDCFA */ - result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, 1, &cqe[0]); -#else result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, /*3 */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); -#endif MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); @@ -457,191 +434,6 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) //MPID_NEM_IB_SEND_PROGRESS_POLLINGSET //} } -#if 0 - else if (req_type == 13 && cqe[i].opcode == IBV_WC_RDMA_READ) { - MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); - - /* end of packet */ - if (req_wrap->mf == MPID_NEM_IB_LMT_LAST_PKT) { - MPIDI_msg_sz_t data_len = req->ch.lmt_data_sz; - MPI_Aint type_size; - - MPID_Datatype_get_size_macro(req->dev.datatype, type_size); - req->dev.recv_data_sz = type_size * req->dev.user_count; - - int complete = 0; - int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); - mpi_errno = - MPIDI_CH3U_Receive_data_found(req, REQ_FIELD(req, lmt_pack_buf), &data_len, - &complete); - - /* Data receive must be completed */ - MPIU_Assert(complete == TRUE); - - MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); - - MPID_nem_ib_lmt_send_PKT_LMT_DONE(req->ch.vc, req); - reqFn = req->dev.OnFinal; - if (reqFn) { - reqFn(req->ch.vc, req, &complete); - } else { - MPIDI_CH3U_Request_complete(req); - } - } - - /* decrement the number of entries in IB command queue */ - vc_ib->ibcom->ncom -= 1; - MPID_nem_ib_ncqe -= 1; - - MPIU_Free(req_wrap); - } - else if (req_type == MPIDI_REQUEST_TYPE_PUT_RESP && cqe[i].opcode == IBV_WC_RDMA_READ) { - MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); - - /* end of packet */ - if (req_wrap->mf == MPID_NEM_IB_LMT_LAST_PKT) { - MPIDI_msg_sz_t data_len = req->ch.lmt_data_sz; - int complete = 0; - mpi_errno = - MPIDI_CH3U_Receive_data_found(req, REQ_FIELD(req, lmt_pack_buf), &data_len, - &complete); - - /* Data receive must be completed */ - MPIU_Assert(complete == TRUE); - - MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); - - complete = 0; - mpi_errno = MPIDI_CH3_ReqHandler_PutRecvComplete(req->ch.vc, req, &complete); // call MPIDI_CH3U_Request_complete() - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - MPIU_Assert(complete == TRUE); - - MPID_nem_ib_lmt_send_PKT_LMT_DONE(req->ch.vc, req); - MPIDI_CH3U_Request_complete(req); - } - - /* decrement the number of entries in IB command queue */ - vc_ib->ibcom->ncom -= 1; - MPID_nem_ib_ncqe -= 1; - - MPIU_Free(req_wrap); - } - else if (req_type == MPIDI_REQUEST_TYPE_PUT_RESP_DERIVED_DT && - cqe[i].opcode == IBV_WC_RDMA_READ) { - MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); - /* end of packet */ - if (req_wrap->mf == MPID_NEM_IB_LMT_LAST_PKT) { - MPIDI_msg_sz_t buflen = req->ch.lmt_data_sz; - char *buf = (char *) REQ_FIELD(req, lmt_pack_buf); - int complete = 0; - int dataloop_size = *(int *) req->dev.dtype_info; /* copy from temp store area */ - - /* copy all of dtype_info and dataloop */ - MPIU_Memcpy(req->dev.dtype_info, buf, sizeof(MPIDI_RMA_dtype_info)); - MPIU_Memcpy(req->dev.dataloop, buf + sizeof(MPIDI_RMA_dtype_info), dataloop_size); - - - /* All dtype data has been received, call req handler */ - mpi_errno = - MPIDI_CH3_ReqHandler_PutDerivedDTRecvComplete(req->ch.vc, req, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_PKT_PUT"); - /* return 'complete == FALSE' */ - - buflen -= (sizeof(MPIDI_RMA_dtype_info) + dataloop_size); - buf += (sizeof(MPIDI_RMA_dtype_info) + dataloop_size); - - mpi_errno = MPID_nem_ib_handle_pkt_bh(req->ch.vc, req, buf, buflen); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_handle_pkt_bh"); - - MPID_nem_ib_lmt_send_PKT_LMT_DONE(req->ch.vc, req); - - MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); - MPIDI_CH3U_Request_complete(req); - } - - /* decrement the number of entries in IB command queue */ - vc_ib->ibcom->ncom -= 1; - MPID_nem_ib_ncqe -= 1; - - MPIU_Free(req_wrap); - } - else if (req_type == MPIDI_REQUEST_TYPE_ACCUM_RESP && cqe[i].opcode == IBV_WC_RDMA_READ) { - MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); - - /* end of packet */ - if (req_wrap->mf == MPID_NEM_IB_LMT_LAST_PKT) { - MPIDI_msg_sz_t data_len = req->ch.lmt_data_sz; - int complete = 0; - mpi_errno = - MPIDI_CH3U_Receive_data_found(req, REQ_FIELD(req, lmt_pack_buf), &data_len, - &complete); - - /* Data receive must be completed */ - MPIU_Assert(complete == TRUE); - - MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); - - complete = 0; - mpi_errno = MPIDI_CH3_ReqHandler_AccumRecvComplete(req->ch.vc, req, &complete); // call MPIDI_CH3U_Request_complete() - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - MPIU_Assert(complete == TRUE); - - MPID_nem_ib_lmt_send_PKT_LMT_DONE(req->ch.vc, req); - MPIDI_CH3U_Request_complete(req); - } - - /* decrement the number of entries in IB command queue */ - vc_ib->ibcom->ncom -= 1; - MPID_nem_ib_ncqe -= 1; - - MPIU_Free(req_wrap); - } - else if (req_type == MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT && - cqe[i].opcode == IBV_WC_RDMA_READ) { - MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); - /* end of packet */ - if (req_wrap->mf == MPID_NEM_IB_LMT_LAST_PKT) { - MPIDI_msg_sz_t buflen = req->ch.lmt_data_sz; - char *buf = (char *) REQ_FIELD(req, lmt_pack_buf); - int complete = 0; - int dataloop_size = *(int *) req->dev.dtype_info; /* copy from temp store area */ - - /* copy all of dtype_info and dataloop */ - MPIU_Memcpy(req->dev.dtype_info, buf, sizeof(MPIDI_RMA_dtype_info)); - MPIU_Memcpy(req->dev.dataloop, buf + sizeof(MPIDI_RMA_dtype_info), dataloop_size); - - - /* All dtype data has been received, call req handler */ - mpi_errno = - MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete(req->ch.vc, req, &complete); - MPIU_ERR_CHKANDJUMP1(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|postrecv", - "**ch3|postrecv %s", "MPIDI_CH3_ACCUMULATE"); - /* return 'complete == FALSE' */ - - buflen -= (sizeof(MPIDI_RMA_dtype_info) + dataloop_size); - buf += (sizeof(MPIDI_RMA_dtype_info) + dataloop_size); - - mpi_errno = MPID_nem_ib_handle_pkt_bh(req->ch.vc, req, buf, buflen); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_handle_pkt_bh"); - - MPID_nem_ib_lmt_send_PKT_LMT_DONE(req->ch.vc, req); - - MPIU_Free(REQ_FIELD(req, lmt_pack_buf)); - MPIDI_CH3U_Request_complete(req); - } - - /* decrement the number of entries in IB command queue */ - vc_ib->ibcom->ncom -= 1; - MPID_nem_ib_ncqe -= 1; - - MPIU_Free(req_wrap); - } -#endif else if (req_type == MPIDI_REQUEST_TYPE_SEND && msg_type == MPIDI_REQUEST_RNDV_MSG && cqe[i].opcode == IBV_WC_RDMA_READ) { MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); @@ -666,11 +458,7 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) else { printf("drain_scq,unknown kind=%d,req_type=%d,msg_type=%d\n", kind, req_type, msg_type); assert(0); -#if 1 // lazy consulting of completion queue MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); -#else - //printf("kind=%d\n", kind); -#endif MPIU_Free(req_wrap); } } @@ -774,13 +562,9 @@ int MPID_nem_ib_drain_scq_scratch_pad() goto fn_exit; } -#if 0 /*def HAVE_LIBDCFA */ - result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, 1, &cqe[0]); -#else result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); -#endif MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { @@ -856,10 +640,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) dprintf("ib_poll_eager,remote_poll=%d,buf=%p,sz=%d\n", *remote_poll, buf, MPID_NEM_IB_NETMOD_HDR_SZ_GET(buf)); -#if 0 - ibcom_errno = MPID_nem_ib_com_poll_cq(MPID_NEM_IB_COM_RC_SHARED_RCQ, &cqe, &result); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_poll_cq"); -#endif dprintf("ib_poll_eager,eager-send,found\n"); fflush(stdout); @@ -879,15 +659,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) int tlb_pref_ahd = (uint64_t) tailmagic + 4096 * MPID_NEM_IB_TLBPREF_POLL - (uint64_t) buf; #endif while (netmod_trailer->tail_flag != MPID_NEM_IB_COM_MAGIC) { - //k++; -#if 0 /* pre-fetch next RDMA-write-buf slot to cover TLB miss latency */ - __asm__ __volatile__ - ("movq %0, %%rsi;" - "movq 0(%%rsi), %%rsi;"::"r"(ringbuf->start + MPID_NEM_IB_COM_RDMABUF_SZSEG * - ((uint16_t) - ((*remote_poll + 1) % MPID_NEM_IB_COM_RDMABUF_NSEG))) - :"%rsi"); -#endif #ifdef MPID_NEM_IB_TLBPREF_POLL __asm__ __volatile__ ("movq %0, %%rsi;" "movq 0(%%rsi), %%rax;"::"r"(buf + tlb_pref_ahd):"%rsi", "%rax"); @@ -906,7 +677,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) * 600 cycle for 512B!!! --> 284 cycle with prefetch */ -#if 1 void *rsi; for (rsi = (void *) buf; rsi < (void *) ((uint8_t *) buf + MPID_NEM_IB_NETMOD_HDR_SZ_GET(buf)); rsi = (uint8_t *) rsi + 64 * 4) { @@ -925,7 +695,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) :"%rsi"); #endif } -#endif /* Increment here because handle_pkt of CLOSE calls poll_eager recursively */ (*remote_poll) += 1; @@ -968,7 +737,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) /* this includes local RDMA-wr-to buf occupation * because MPID_nem_handle_pkt releases RDMA-wr-to buf by copying data out */ /* responder releases resource and then embed largest sequence number into MPI message bound to initiator */ -#if 1 if ((vc->state != MPIDI_VC_STATE_INACTIVE) || (vc->state == MPIDI_VC_STATE_INACTIVE && vc_ib->vc_terminate_buf == buf)) { dprintf @@ -1031,7 +799,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) } } } -#endif if ((vc->state != MPIDI_VC_STATE_INACTIVE) || (vc->state == MPIDI_VC_STATE_INACTIVE && vc_ib->vc_terminate_buf == buf)) { @@ -1049,7 +816,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) (vc->state == MPIDI_VC_STATE_INACTIVE && vc_ib->vc_terminate_buf == buf)) MPID_NEM_IB_NETMOD_HDR_HEAD_FLAG_SET(buf, 0); -#if 1 /* We move this code from the end of vc_terminate. */ if (vc->state == MPIDI_VC_STATE_INACTIVE && vc_ib->vc_terminate_buf == buf) { /* clear stored data */ vc_ib->vc_terminate_buf = NULL; @@ -1095,7 +861,6 @@ int MPID_nem_ib_poll_eager(MPID_nem_ib_ringbuf_t * ringbuf) MPIU_Free(MPID_nem_ib_scratch_pad_ibcoms); } } -#endif fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_POLL_EAGER); @@ -1118,9 +883,7 @@ int MPID_nem_ib_poll(int in_blocking_poll) MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_POLL); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_POLL); -#if 1 unsigned int progress_completion_count_old = MPIDI_CH3I_progress_completion_count.v; -#endif /* poll lmt */ /* when receiver side sends CTS to sender side @@ -1200,16 +963,9 @@ int MPID_nem_ib_poll(int in_blocking_poll) MPI_ERR_TYPE, "**MPID_nem_ib_poll", 0); /* --END ERROR HANDLING-- */ } -#if 1 /* debug, enable again later, polling waits forever when freeing it here. */ - //if (rreq->ref_count == 1) { dprintf("ib_poll,lmt,ref_count=%d,lmt_pack_buf=%p\n", rreq->ref_count, REQ_FIELD(rreq, lmt_pack_buf)); - //MPIU_Free(REQ_FIELD(rreq, lmt_pack_buf)); MPID_nem_ib_stfree(REQ_FIELD(rreq, lmt_pack_buf), (size_t) rreq->ch.lmt_data_sz); - //} else { - // dprintf("ib_poll,lmt,ref_count=%d,lmt_pack_buf=%p\n", rreq->ref_count, REQ_FIELD(rreq, lmt_pack_buf)); - //} -#endif } /* send done to sender. vc is stashed in MPID_nem_ib_lmt_start_recv (in ib_lmt.c) */ @@ -1309,12 +1065,6 @@ int MPID_nem_ib_poll(int in_blocking_poll) (VC_FIELD(MPID_nem_ib_ringbuf[i].vc, ibcom->ncom) >= MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN); } - - -#if 0 - /* aggressively perform drain_scq */ - ncom_almost_full |= !(MPID_nem_ib_sendq_empty(VC_FIELD(MPID_nem_ib_ringbuf[i].vc, sendq))); -#endif } #if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { @@ -1346,11 +1096,10 @@ int MPID_nem_ib_poll(int in_blocking_poll) } #endif } -#if 1 /* aggressively perform drain_scq */ ibcom_errno = MPID_nem_ib_drain_scq(0); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); -#endif + #ifdef MPID_NEM_IB_ONDEMAND /* process incoming connection request */ MPID_nem_ib_cm_poll_syn(); @@ -1369,13 +1118,11 @@ int MPID_nem_ib_poll(int in_blocking_poll) #endif MPID_nem_ib_ringbuf_progress(); MPID_nem_ib_progress_engine_vt += 1; /* Progress virtual time */ -#if 1 + /* if polling on eager-send and lmt would repeat frequently, perform "pause" to yield instruction issue bandwitdh to other logical-core */ if (in_blocking_poll && progress_completion_count_old == MPIDI_CH3I_progress_completion_count.v) { __asm__ __volatile__("pause;":::"memory"); } -#endif - //if (in_blocking_poll) { goto prev; } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_POLL); @@ -1406,12 +1153,6 @@ int MPID_nem_ib_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) } #endif -#if 0 - int ibcom_errno; - ibcom_errno = MPID_nem_ib_com_irecv(vc_ib->sc->fd, (uint64_t) vc->pg_rank); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_irecv"); -#endif -#if 1 /*takagi */ MPIDI_msg_sz_t data_sz; int dt_contig; MPI_Aint dt_true_lb; @@ -1422,37 +1163,23 @@ int MPID_nem_ib_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) /* anticipating received message finds maching request in the posted-queue */ if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) > vc->eager_max_msg_sz) { //if (MPID_nem_ib_tsc_poll - MPID_nem_ib_rdtsc() > MPID_NEM_IB_POLL_PERIOD_RECV_POSTED) { -//#if 1 if (VC_FIELD(vc, ibcom->remote_ringbuf)) { mpi_errno = MPID_nem_ib_poll_eager(VC_FIELD(vc, ibcom->remote_ringbuf)); -//#else -// mpi_errno = MPID_nem_ib_poll(0); -//#endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } } - //} } else { -#if 1 /* anticipating received message finds maching request in the posted-queue */ - //if (MPID_nem_ib_tsc_poll - MPID_nem_ib_rdtsc() > MPID_NEM_IB_POLL_PERIOD_RECV_POSTED) { if (VC_FIELD(vc, ibcom->remote_ringbuf)) { -#if 1 mpi_errno = MPID_nem_ib_poll_eager(VC_FIELD(vc, ibcom->remote_ringbuf)); -#else - mpi_errno = MPID_nem_ib_poll(0); -#endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } } - //} -#endif } -#endif fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_RECV_POSTED); @@ -1480,7 +1207,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_ib_myrank, vc->pg_rank, user_data); -#if 1 /* moving from ib_poll */ + /* Clear all possible tail flag slots */ /* tail flag is located at MPID_NEM_IB_COM_INLINE_DATA boundary and variable length entails multiple prospective locations for the future use */ /* see MPIDI_CH3_PktHandler_EagerShortSend (in src/mpid/ch3/src/ch3u_eager.c */ @@ -1506,7 +1233,6 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) int off_pow2_aligned; MPID_NEM_IB_OFF_POW2_ALIGNED(MPID_NEM_IB_NETMOD_HDR_SZ_GET(buf)); //dprintf("recv_buf_released,sz=%d,pow2=%d\n", MPID_NEM_IB_NETMOD_HDR_SZ_GET(buf), off_pow2_aligned); -#if 1 uint32_t offset; for (offset = 15;; offset = @@ -1524,10 +1250,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) break; } } -#endif -#endif -#if 1 /* moving from ib_poll */ /* mark that one eager-send RDMA-write-to buffer has been released */ uint16_t index_slot = (unsigned long) ((uint8_t *) user_data - @@ -1625,7 +1348,6 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) MPID_nem_ib_send_reply_seq_num(vc); skip:; } -#endif fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); @@ -1634,54 +1356,6 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) goto fn_exit; } -#if 0 -/* packet handler for wrapper packet of MPIDI_NEM_PKT_LMT_DONE */ -/* see pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_PktHandler_lmt_done -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) -{ - int mpi_errno = MPI_SUCCESS; - int ibcom_errno; - MPID_nem_ib_pkt_lmt_done_t *const done_pkt = (MPID_nem_ib_pkt_lmt_done_t *) pkt; - MPID_Request *req; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_LMT_DONE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_LMT_DONE); - /* Check the assumption on sizeof(MPIDI_CH3_Pkt_t). - * It is utilized in pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) - * that must be larger than sizeof(MPID_nem_ib_pkt_lmt_done_t) */ - if (sizeof(MPID_nem_ib_pkt_lmt_done_t) > sizeof(MPIDI_CH3_Pkt_t)) { - MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_INTERN, "**sizeof(MPIDI_CH3_Pkt_t)"); - } - - /* fall back to the original handler */ - /* we don't need to worry about the difference caused by embedding seq_num - * because the handler does not use it (e.g. applying sizeof operator to it) */ - MPID_nem_pkt_lmt_done_t *pkt_parent_class = (MPID_nem_pkt_lmt_done_t *) pkt; - pkt_parent_class->type = MPIDI_NEM_PKT_LMT_DONE; -#if 0 - mpi_errno = MPID_nem_handle_pkt(vc, (char *) pkt_parent_class, *buflen); -#else - MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**notimplemented"); - /* you need to modify mpid_nem_lmt.c to make pkt_DONE_handler visible to me */ - //mpi_errno = pkt_DONE_handler(vc, pkt, buflen, rreqp); -#endif - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_LMT_DONE); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif - /* packet handler for wrapper packet of MPIDI_CH3_PKT_EAGER_SEND */ /* see MPIDI_CH3_PktHandler_EagerSend (in src/mpid/ch3/src/ch3u_eager.c) */ #undef FUNCNAME @@ -1740,16 +1414,12 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, * see MPID_nem_ib_iSendContig */ //ch3_pkt->type = MPIDI_CH3_PKT_EAGER_SEND; -#if 0 - mpi_errno = MPID_nem_handle_pkt(vc, (char *) pkt_parent_class, *buflen); -#else dprintf("ib_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_EagerSend(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); dprintf("ib_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen); *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); dprintf("ib_poll.c,after addition,buflen=%ld\n", *buflen); -#endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1761,70 +1431,6 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, goto fn_exit; } -#if 0 /* modification of mpid_nem_lmt.c is required */ - -/* Temporary fix because it's static */ -int pkt_RTS_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); -/* packet handler for wrapper packet of MPIDI_NEM_PKT_LMT_RTS */ -/* see pkt_RTS_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_pkt_RTS_handler -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_pkt_RTS_handler(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) -{ - MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; - MPIDI_CH3_Pkt_t *ch3_pkt = - (MPIDI_CH3_Pkt_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); - int mpi_errno = MPI_SUCCESS; - int ibcom_errno; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKT_RTS_HANDLER); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKT_RTS_HANDLER); - /* Update occupation status of local SR (send request) queue */ - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - dprintf - ("MPID_nem_ib_pkt_RTS_handler,lsr_seq_num_tail=%d,netmod_pkt->seq_num_tail=%d\n", - vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - vc_ib->ibcom->lsr_seq_num_tail = netmod_pkt->seq_num_tail; - dprintf - ("MPID_nem_ib_pkt_RTS_handler,lsr_seq_num_tail updated to %d\n", - vc_ib->ibcom->lsr_seq_num_tail); -#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE - /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,rts,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); - dprintf("pkthandler,rts,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); -#endif - dprintf("pkthandler,rts,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, - MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); - /* try to send from sendq because at least one RDMA-write-to buffer has been released */ - dprintf("pkthandler,eagersend,send_progress\n"); - fflush(stdout); - MPID_NEM_IB_CHECK_AND_SEND_PROGRESS; - /* fall back to the original handler */ - /* we don't need to worry about the difference caused by embedding seq_num - * because size of MPI-header of MPIDI_CH3_PKT_EAGER_SEND equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_ib_iSendContig - */ - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); - mpi_errno = pkt_RTS_handler(vc, ch3_pkt, &ch3_buflen, rreqp); - *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKT_RTS_HANDLER); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif - #undef FUNCNAME #define FUNCNAME MPID_nem_ib_PktHandler_rma_lmt_rts #undef FCNAME @@ -1915,443 +1521,6 @@ int MPID_nem_ib_PktHandler_rma_lmt_rts(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, goto fn_exit; } -#if 0 -/* packet handler for wrapper packet of MPIDI_CH3_PKT_PUT */ -/* see MPIDI_CH3_PktHandler_EagerSend (in src/mpid/ch3/src/ch3u_rma_sync.c) */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_PktHandler_Put -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) -{ - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - int mpi_errno = MPI_SUCCESS; - MPID_Request *req = NULL; - MPIDI_CH3_Pkt_put_t *put_pkt = - (MPIDI_CH3_Pkt_put_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t)); - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = - (MPID_nem_ib_rma_lmt_cookie_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t) + - sizeof(MPIDI_CH3_Pkt_t)); - - /* ref. MPIDI_CH3_PktHandler_Put (= pktArray[MPIDI_CH3_PKT_PUT]) */ - MPI_Aint type_size; - - MPID_Win *win_ptr; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_PUT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_PUT); - - MPIU_Assert(put_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr); - - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); /* decrement only in drain_scq ? */ - int incomplete; - MPIDI_CH3U_Request_increment_cc(req, &incomplete); // decrement in drain_scq - - req->dev.user_buf = put_pkt->addr; - req->dev.user_count = put_pkt->count; - req->dev.target_win_handle = put_pkt->target_win_handle; - req->dev.source_win_handle = put_pkt->source_win_handle; - req->dev.flags = put_pkt->flags; - req->dev.OnFinal = MPIDI_CH3_ReqHandler_PutRecvComplete; - - if (MPIR_DATATYPE_IS_PREDEFINED(put_pkt->datatype)) { - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP); - req->dev.datatype = put_pkt->datatype; - - MPID_Datatype_get_size_macro(put_pkt->datatype, type_size); - req->dev.recv_data_sz = type_size * put_pkt->count; - if (put_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, put_pkt->data, put_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + put_pkt->immed_len); - req->dev.recv_data_sz -= put_pkt->immed_len; - } - } - else { - /* derived datatype */ - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_PUT_RESP_DERIVED_DT); - req->dev.datatype = MPI_DATATYPE_NULL; - - req->dev.dtype_info = (MPIDI_RMA_dtype_info *) MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); - req->dev.dataloop = MPIU_Malloc(put_pkt->dataloop_size); - - /* We have to store the value of 'put_pkt->dataloop_size' which we use in drain_scq. - * Temporarily, put it in req->dev.dtype_info. - */ - *(int *) req->dev.dtype_info = put_pkt->dataloop_size; - } - - /* ref. pkt_RTS_handler (= pktArray[MPIDI_NEM_PKT_LMT_RTS]) */ - - void *write_to_buf; - - req->ch.lmt_data_sz = s_cookie_buf->len; - req->ch.lmt_req_id = s_cookie_buf->sender_req_id; - - REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc((size_t) req->ch.lmt_data_sz); - write_to_buf = REQ_FIELD(req, lmt_pack_buf); - - /* stash vc for ib_poll */ - req->ch.vc = vc; - - REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail; - - /* try to issue RDMA-read command */ - int slack = 1; /* slack for control packet bringing sequence number */ - if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && - vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && - MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { - mpi_errno = - MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, - s_cookie_buf->len, write_to_buf, - s_cookie_buf->max_msg_sz, 1); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } - else { - /* enqueue command into send_queue */ - dprintf("lmt_start_recv, enqueuing,sendq_empty=%d,ncom=%d,ncqe=%d\n", - MPID_nem_ib_sendq_empty(vc_ib->sendq), - vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, - MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY); - - /* make raddr, (sz is in rreq->ch.lmt_data_sz), rkey, (user_buf is in req->dev.user_buf) survive enqueue, free cookie, dequeue */ - REQ_FIELD(req, lmt_raddr) = s_cookie_buf->addr; - REQ_FIELD(req, lmt_rkey) = s_cookie_buf->rkey; - REQ_FIELD(req, lmt_write_to_buf) = write_to_buf; - REQ_FIELD(req, lmt_szsend) = s_cookie_buf->len; - REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; - REQ_FIELD(req, last) = 1; /* not support segmentation */ - - /* set for send_progress */ - MPIDI_Request_set_msg_type(req, MPIDI_REQUEST_RNDV_MSG); - req->kind = MPID_REQUEST_RECV; - - MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req); - } - - /* prefix + header + data */ - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_rma_lmt_cookie_t); - *rreqp = NULL; - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_PUT); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif - -#if 0 -/* packet handler for wrapper packet of MPIDI_CH3_PKT_ACCUMULATE */ -/* see MPIDI_CH3_PktHandler_Accumulate (in src/mpid/ch3/src/ch3u_rma_sync.c) */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_PktHandler_Accumulate -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) -{ - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - int mpi_errno = MPI_SUCCESS; - MPID_Request *req = NULL; - MPIDI_CH3_Pkt_accum_t *accum_pkt = - (MPIDI_CH3_Pkt_accum_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t)); - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = - (MPID_nem_ib_rma_lmt_cookie_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t) + - sizeof(MPIDI_CH3_Pkt_t)); - - /* ref. MPIDI_CH3_PktHandler_Accumulate */ - MPI_Aint true_lb, true_extent, extent; - MPI_Aint type_size; - MPID_Win *win_ptr; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_ACCUMULATE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_ACCUMULATE); - - MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL); - MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr); - - req = MPID_Request_create(); - MPIU_Object_set_ref(req, 1); - - int incomplete; - MPIDI_CH3U_Request_increment_cc(req, &incomplete); // decrement in drain_scq - - req->dev.user_count = accum_pkt->count; - req->dev.op = accum_pkt->op; - req->dev.real_user_buf = accum_pkt->addr; - req->dev.target_win_handle = accum_pkt->target_win_handle; - req->dev.source_win_handle = accum_pkt->source_win_handle; - req->dev.flags = accum_pkt->flags; - - req->dev.resp_request_handle = MPI_REQUEST_NULL; - req->dev.OnFinal = MPIDI_CH3_ReqHandler_AccumRecvComplete; - - if (MPIR_DATATYPE_IS_PREDEFINED(accum_pkt->datatype)) { - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP); - req->dev.datatype = accum_pkt->datatype; - - MPIR_Type_get_true_extent_impl(accum_pkt->datatype, &true_lb, &true_extent); - MPID_Datatype_get_extent_macro(accum_pkt->datatype, extent); - - /* Predefined types should always have zero lb */ - MPIU_Assert(true_lb == 0); - - req->dev.user_buf = MPIU_Malloc(accum_pkt->count * (MPIR_MAX(extent, true_extent))); - req->dev.final_user_buf = req->dev.user_buf; - - MPID_Datatype_get_size_macro(accum_pkt->datatype, type_size); - req->dev.recv_data_sz = type_size * accum_pkt->count; - - if (accum_pkt->immed_len > 0) { - /* See if we can receive some data from packet header. */ - MPIU_Memcpy(req->dev.user_buf, accum_pkt->data, accum_pkt->immed_len); - req->dev.user_buf = (void*)((char*)req->dev.user_buf + accum_pkt->immed_len); - req->dev.recv_data_sz -= accum_pkt->immed_len; - } - - } - else { - MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_ACCUM_RESP_DERIVED_DT); - req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_AccumDerivedDTRecvComplete; - req->dev.datatype = MPI_DATATYPE_NULL; - - req->dev.dtype_info = (MPIDI_RMA_dtype_info *) MPIU_Malloc(sizeof(MPIDI_RMA_dtype_info)); - req->dev.dataloop = MPIU_Malloc(accum_pkt->dataloop_size); - - /* We have to store the value of 'put_pkt->dataloop_size' which we use in drain_scq. - * Temporarily, put it in req->dev.dtype_info. - */ - *(int *) req->dev.dtype_info = accum_pkt->dataloop_size; - } - - /* ref. pkt_RTS_handler (= pktArray[MPIDI_NEM_PKT_LMT_RTS]) */ - void *write_to_buf; - - req->ch.lmt_data_sz = s_cookie_buf->len; - req->ch.lmt_req_id = s_cookie_buf->sender_req_id; - - REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc((size_t) req->ch.lmt_data_sz); - write_to_buf = REQ_FIELD(req, lmt_pack_buf); - - /* stash vc for ib_poll */ - req->ch.vc = vc; - - REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail; - - /* try to issue RDMA-read command */ - int slack = 1; /* slack for control packet bringing sequence number */ - if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && - vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && - MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { - mpi_errno = - MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, - s_cookie_buf->len, write_to_buf, - s_cookie_buf->max_msg_sz, 1); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } - else { - /* enqueue command into send_queue */ - dprintf("lmt_start_recv, enqueuing,sendq_empty=%d,ncom=%d,ncqe=%d\n", - MPID_nem_ib_sendq_empty(vc_ib->sendq), - vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, - MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY); - - /* make raddr, (sz is in rreq->ch.lmt_data_sz), rkey, (user_buf is in req->dev.user_buf) survive enqueue, free cookie, dequeue */ - REQ_FIELD(req, lmt_raddr) = s_cookie_buf->addr; - REQ_FIELD(req, lmt_rkey) = s_cookie_buf->rkey; - REQ_FIELD(req, lmt_write_to_buf) = write_to_buf; - REQ_FIELD(req, lmt_szsend) = s_cookie_buf->len; - REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; - REQ_FIELD(req, last) = 1; /* not support segmentation */ - - /* set for send_progress */ - MPIDI_Request_set_msg_type(req, MPIDI_REQUEST_RNDV_MSG); - req->kind = MPID_REQUEST_RECV; - - MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req); - } - - /* prefix + header + data */ - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_rma_lmt_cookie_t); - *rreqp = NULL; - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_ACCUMULATE); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif - -#if 0 -/* packet handler for wrapper packet of MPIDI_CH3_PKT_GET */ -/* see MPIDI_CH3_PktHandler_Get (in src/mpid/ch3/src/ch3u_rma_sync.c) */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_PktHandler_Get -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) -{ - MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; - MPIDI_CH3_Pkt_get_t *ch3_pkt = - (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GET); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GET); - /* Update occupation status of local SR (send request) queue */ - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - dprintf - ("MPID_nem_ib_Pkthandler_Get,lsr_seq_num_tail=%d,get_pkt->seq_num_tail=%d\n", - vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - vc_ib->ibcom->lsr_seq_num_tail = netmod_pkt->seq_num_tail; - dprintf("MPID_nem_ib_Pkthandler_Get,lsr_seq_num_tail updated to %d\n", - vc_ib->ibcom->lsr_seq_num_tail); -#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE - /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); - dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); -#endif - dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, - MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); - /* try to send from sendq because at least one RDMA-write-to buffer has been released */ - dprintf("pkthandler,get,send_progress\n"); - fflush(stdout); - MPID_NEM_IB_SEND_PROGRESS_POLLINGSET - /* fall back to the original handler */ - /* we don't need to worry about the difference caused by embedding seq_num - * because size of MPI-header of MPIDI_CH3_PKT_PUT equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_ib_iSendContig - */ - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); - mpi_errno = MPIDI_CH3_PktHandler_Get(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GET); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -/* packet handler for wrapper packet of MPIDI_CH3_PKT_GET_RESP */ -/* see MPIDI_CH3_PktHandler_GetResp (in src/mpid/ch3/src/ch3u_rma_sync.c) */ -#undef FUNCNAME -#define FUNCNAME MPID_nem_ib_PktHandler_GetResp -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) -{ - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - int mpi_errno = MPI_SUCCESS; - MPID_Request *req = NULL; - MPIDI_CH3_Pkt_get_resp_t *get_resp_pkt = - (MPIDI_CH3_Pkt_get_resp_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t)); - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = - (MPID_nem_ib_rma_lmt_cookie_t *) ((uint8_t *) pkt + sizeof(MPIDI_CH3_Pkt_t) + - sizeof(MPIDI_CH3_Pkt_t)); - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GETRESP); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GETRESP); - - MPID_Request_get_ptr(get_resp_pkt->request_handle, req); - - MPID_Win *win_ptr; - int target_rank = get_resp_pkt->target_rank; - - MPID_Win_get_ptr(get_resp_pkt->source_win_handle, win_ptr); - - /* decrement ack_counter on target */ - if (get_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED) { - mpi_errno = handle_lock_ack(win_ptr, target_rank, - get_resp_pkt->flags); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } - if (get_resp_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK) { - mpi_errno = MPIDI_CH3I_RMA_Handle_flush_ack(win_ptr, target_rank); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - } - - void *write_to_buf; - - req->ch.lmt_data_sz = s_cookie_buf->len; - req->ch.lmt_req_id = s_cookie_buf->sender_req_id; - - REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc((size_t) req->ch.lmt_data_sz); - write_to_buf = REQ_FIELD(req, lmt_pack_buf); - - /* This is magic number to pick up request in drain_scq */ - MPIDI_Request_set_type(req, 13); // currently Request-type is defined from 1 to 12. - - /* stash vc for ib_poll */ - req->ch.vc = vc; - - REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail; - - /* try to issue RDMA-read command */ - int slack = 1; /* slack for control packet bringing sequence number */ - if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && - vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && - MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { - mpi_errno = - MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, - s_cookie_buf->len, write_to_buf, - s_cookie_buf->max_msg_sz, 1); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } - else { - /* enqueue command into send_queue */ - dprintf("lmt_start_recv, enqueuing,sendq_empty=%d,ncom=%d,ncqe=%d\n", - MPID_nem_ib_sendq_empty(vc_ib->sendq), - vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, - MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY); - - /* make raddr, (sz is in rreq->ch.lmt_data_sz), rkey, (user_buf is in req->dev.user_buf) survive enqueue, free cookie, dequeue */ - REQ_FIELD(req, lmt_raddr) = s_cookie_buf->addr; - REQ_FIELD(req, lmt_rkey) = s_cookie_buf->rkey; - REQ_FIELD(req, lmt_write_to_buf) = write_to_buf; - REQ_FIELD(req, lmt_szsend) = s_cookie_buf->len; - REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; - REQ_FIELD(req, last) = 1; /* not support segmentation */ - - MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req); - } - - /* prefix + header + data */ - *buflen = - sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_rma_lmt_cookie_t); - *rreqp = NULL; - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GETRESP); - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif - /* MPI_Isend set req-type to MPIDI_REQUEST_TYPE_RECV */ #undef FUNCNAME #define FUNCNAME MPID_nem_ib_pkt_GET_DONE_handler @@ -2381,23 +1550,6 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, case MPIDI_REQUEST_TYPE_RSEND: case MPIDI_REQUEST_TYPE_SSEND: case MPIDI_REQUEST_TYPE_BSEND: -#if 0 /* obsolete, it's in netmod header now */ - /* extract embeded RDMA-write-to buffer occupancy information */ - dprintf - ("get_done_handler,old lsr_seq_num_tail=%d,done_pkt->seq_num_tail=%d\n", - vc_ib->ibcom->lsr_seq_num_tail, done_pkt->seq_num_tail); - vc_ib->ibcom->lsr_seq_num_tail = done_pkt->seq_num_tail; - //dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail); -#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE - /* change remote notification policy of RDMA-write-to buf */ - //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); - //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); -#endif - //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); -#endif - /* decrement reference counter of mr_cache_entry */ MPID_nem_ib_com_reg_mr_release(REQ_FIELD(req, lmt_mr_cache)); @@ -2773,7 +1925,6 @@ int MPID_nem_ib_cm_drain_scq() dprintf("cm_drain_scq,cm_cas,succeeded\n"); if (is_conn_established(shadow_cm->req->responder_rank)) { -#if 1 /* Explicitly release CAS word because * ConnectX-3 doesn't support safe CAS with PCI device and CPU */ MPID_nem_ib_cm_cas_release(MPID_nem_ib_conns @@ -2789,49 +1940,6 @@ int MPID_nem_ib_cm_drain_scq() /* free memory : req->ref_count is 3, so call MPIU_Free() directly */ //MPID_nem_ib_cm_request_release(shadow_cm->req); MPIU_Free(shadow_cm->req); -#else - /* Connection is already established. - * In this case, responder may already have performed vc_terminate. - * However, since initiator has to release responder's CAS word, - * initiator sends CM_CAS_RELEASE2. */ - dprintf("cm_drain_scq,cm_cas,established,%d->%d\n", - MPID_nem_ib_myrank, shadow_cm->req->responder_rank); - shadow_cm->req->state = MPID_NEM_IB_CM_CAS_RELEASE2; - if (MPID_nem_ib_ncqe_scratch_pad < MPID_NEM_IB_COM_MAX_CQ_CAPACITY && - shadow_cm->req->ibcom->ncom_scratch_pad < - MPID_NEM_IB_COM_MAX_SQ_CAPACITY) { - - MPID_nem_ib_cm_cmd_syn_t *cmd = - (MPID_nem_ib_cm_cmd_syn_t *) shadow_cm->req->ibcom-> - icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM]; - MPID_NEM_IB_CM_COMPOSE_CAS_RELEASE2(cmd, shadow_cm->req); - cmd->initiator_rank = MPID_nem_ib_myrank; - MPID_nem_ib_cm_cmd_shadow_t *shadow_syn = - (MPID_nem_ib_cm_cmd_shadow_t *) - MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); - shadow_syn->type = shadow_cm->req->state; - shadow_syn->req = shadow_cm->req; - dprintf("shadow_syn=%p,shadow_syn->req=%p\n", shadow_syn, - shadow_syn->req); - dprintf("cm_drain_scq,cm_cas,established,sending cas_release2,%d->%d\n", - MPID_nem_ib_myrank, shadow_cm->req->responder_rank); - mpi_errno = - MPID_nem_ib_cm_cmd_core(shadow_cm->req->responder_rank, shadow_syn, - (void *) cmd, - sizeof(MPID_nem_ib_cm_cmd_syn_t), - 1 /* syn:1 */ , 0); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_cm_send_core"); - } - else { - MPID_NEM_IB_CM_COMPOSE_CAS_RELEASE2((MPID_nem_ib_cm_cmd_syn_t *) & - (shadow_cm->req->cmd), - shadow_cm->req); - ((MPID_nem_ib_cm_cmd_syn_t *) & shadow_cm->req->cmd)->initiator_rank = - MPID_nem_ib_myrank; - MPID_nem_ib_cm_sendq_enqueue(&MPID_nem_ib_cm_sendq, shadow_cm->req); - } -#endif } else { /* Increment receiving transaction counter. Initiator receives SYNACK and ACK2 */ @@ -3879,14 +2987,6 @@ int MPID_nem_ib_ringbuf_alloc(MPIDI_VC_t * vc) MPIU_ERR_CHKANDJUMP(!MPID_nem_ib_ringbuf, mpi_errno, MPI_ERR_OTHER, "**malloc"); } -#if 0 /* Degug, "#if 1" to make exclusive ring-buffers not available */ - //if (MPID_nem_ib_myrank == 0) { - for (i = 0; i < MPID_NEM_IB_NRINGBUF - 1; i++) { - MPID_nem_ib_ringbuf_acquired[i / 64] |= (1ULL << (i & 63)); - } - //} -#endif - int found = 0; /* [MPID_NEM_IB_NRINGBUF-1] holds shared ring buffer */ for (i = 0; i < MPID_NEM_IB_NRINGBUF - 1; i++) { diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c index 2a2c8b6..bd8b5f5 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c @@ -34,10 +34,6 @@ static int ref_count; typedef struct { char *next; } free_list_t; -#if 0 -static char *free_list_front[MPID_NEM_IB_NIALLOCID] = { 0 }; -static char *arena_flist[MPID_NEM_IB_NIALLOCID] = { 0 }; -#endif #define MPID_NEM_IB_SZARENA 4096 #define MPID_NEM_IB_CLUSTER_SIZE (MPID_NEM_IB_SZARENA/sz) @@ -155,35 +151,16 @@ static inline void __lru_queue_display() void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, enum ibv_access_flags additional_flags, int mode) { -#if 0 /* debug */ - struct ibv_mr *mr; - int ibcom_errno = MPID_nem_ib_com_reg_mr(addr, len, &mr); - printf("mrcache,MPID_nem_ib_com_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len, - mr->lkey, mr->rkey); - if (ibcom_errno != 0) { - goto fn_fail; - } - fn_exit: - return mr; - fn_fail: - goto fn_exit; -#else int ibcom_errno; int key; struct MPID_nem_ib_com_reg_mr_cache_entry_t *e; static unsigned long long num_global_cache = 0ULL; -#if 1 /*def HAVE_LIBDCFA */ /* we can't change addr because ibv_post_send assumes mr->host_addr (output of this function) * must have an exact mirror value of addr (input of this function) */ void *addr_aligned = addr; long len_aligned = len; -#else - void *addr_aligned = (void *) ((unsigned long) addr & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); - long len_aligned = - ((((unsigned long) addr + len) - (unsigned long) addr_aligned + - MPID_NEM_IB_COM_REG_MR_SZPAGE - 1) & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); -#endif + key = MPID_nem_ib_com_hash_func(addr); dprintf("[MrCache] addr=%p, len=%ld\n", addr, len); @@ -208,24 +185,6 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, // miss -#if 0 - // evict an entry and de-register its MR when the cache-set is full - if (way > MPID_NEM_IB_COM_REG_MR_NWAY) { - struct MPID_nem_ib_com_reg_mr_cache_entry_t *victim = - (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) e->lru_prev; - MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *) victim); - - //dprintf("MPID_nem_ib_com_reg_mr,evict,entry addr=%p,len=%d,mr addr=%p,len=%ld\n", e->addr, e->len, - //e->mr->addr, e->mr->length); - ibcom_errno = MPID_nem_ib_com_dereg_mr(victim->mr); - if (ibcom_errno) { - printf("mrcache,MPID_nem_ib_com_dereg_mr\n"); - goto fn_fail; - } - afree(victim, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); - } -#endif - e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t), MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); /* reference counter is used when evicting entry */ @@ -237,7 +196,6 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, if (ibcom_errno != 0) { /* ib_com_reg_mr returns the errno of ibv_reg_mr */ if (ibcom_errno == ENOMEM) { -#if 1 /* deregister memory region unused and re-register new one */ struct MPID_nem_ib_com_reg_mr_listnode_t *ptr; struct MPID_nem_ib_com_reg_mr_cache_entry_t *victim; @@ -287,40 +245,6 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, afree(e, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); goto fn_fail; } -#else - /* deregister memory region. The value of 'num_global_cache' means the number of global-cached. - * delete 5 percents of global-cached */ - int i; - int del_num = (num_global_cache + 19) / 20; - struct MPID_nem_ib_com_reg_mr_cache_entry_t *victim; - - dprintf("mrcache,MPID_nem_ib_com_reg_mr,ENOMEM,del_num(%d)\n", del_num); - - for (i = 0; i < del_num; i++) { - /* get LRU data from MPID_nem_ib_com_reg_mr_global_cache */ - victim = list_entry(MPID_nem_ib_com_reg_mr_global_cache.lru_prev, struct MPID_nem_ib_com_reg_mr_cache_entry_t, g_lru); - - MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *)victim); - MPID_nem_ib_com_reg_mr_unlink(&(victim->g_lru)); - - ibcom_errno = MPID_nem_ib_com_dereg_mr(victim->mr); - if (ibcom_errno) { - printf("mrcache,MPID_nem_ib_com_dereg_mr\n"); - afree(e, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); - goto fn_fail; - } - afree(victim, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); - num_global_cache--; - } - - /* re-registraion */ - ibcom_errno = MPID_nem_ib_com_reg_mr(addr_aligned, len_aligned, &e->mr, additional_flags); - if (ibcom_errno != 0) { - fprintf(stderr, "mrcache,MPID_nem_ib_com_reg_mr,retry,errno=%d\n", ibcom_errno); - afree(e, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); - goto fn_fail; - } -#endif } else { /* errno is not ENOMEM */ @@ -352,7 +276,7 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, /* reference counter is used when evicting entry */ e->refc++; -#if 1 + /* move to head of the list */ if (e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next) @@ -361,7 +285,7 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); } -#endif + if (mode != MPID_NEM_IB_COM_REG_MR_STICKY) { /* move to head of the list in global-cache */ MPID_nem_ib_com_reg_mr_unlink(&(e->g_lru)); @@ -379,23 +303,8 @@ void *MPID_nem_ib_com_reg_mr_fetch(void *addr, long len, return e; fn_fail: goto fn_exit; -#endif } -#if 0 -static void MPID_nem_ib_com_reg_mr_dereg(struct ibv_mr *mr) -{ - - struct MPID_nem_ib_com_reg_mr_cache_entry_t *e; - struct MPID_nem_ib_com_reg_mr_cache_entry_t *zero = 0; - unsigned long offset = (unsigned long) zero->mr; - e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) ((unsigned long) mr - offset); - e->refc--; - - //dprintf("MPID_nem_ib_com_reg_mr_dereg,entry=%p,mr=%p,addr=%p,refc=%d,offset=%lx\n", e, mr, e->mr->addr, - //e->refc, offset); -} -#endif void MPID_nem_ib_com_reg_mr_release(struct MPID_nem_ib_com_reg_mr_cache_entry_t *entry) { entry->refc--; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index fe43e85..8981409 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -19,128 +19,6 @@ static int entered_send_progress = 0; -#ifdef MPID_NEM_IB_ONDEMAND -#if 0 -// tree format is -// one or more <left_pointer(int), right_pointer(int), value(int), length(int), string(char[])> -#define MPID_NEM_IB_MAP_LPTR(ptr) *(int*)((ptr) + sizeof(int)*0) -#define MPID_NEM_IB_MAP_RPTR(ptr) *(int*)((ptr) + sizeof(int)*1) -#define MPID_NEM_IB_MAP_VAL(ptr) *(int*)((ptr) + sizeof(int)*2) -#define MPID_NEM_IB_MAP_LEN(ptr) *(int*)((ptr) + sizeof(int)*3) -#define MPID_NEM_IB_MAP_PBODY(ptr) ((ptr) + sizeof(int)*4) - -#define MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, initial) { \ - if (map->length + key_length + sizeof(int)*4 > map->max_length) { \ - map->max_length = map->max_length ? map->max_length * 2 : 4096; \ - map->data = realloc(map->data, map->max_length); \ - } \ - char* new_str = map->data + map->length; \ - MPID_NEM_IB_MAP_LPTR(new_str) = 0; \ - MPID_NEM_IB_MAP_RPTR(new_str) = 0; \ - MPID_NEM_IB_MAP_VAL(new_str) = initial; \ - MPID_NEM_IB_MAP_LEN(new_str) = key_length; \ - memcpy(MPID_NEM_IB_MAP_PBODY(new_str), key, key_length); \ - map->length += sizeof(int)*4 + key_length; \ -} - -void MPID_nem_ib_cm_map_set(MPID_nem_ib_cm_map_t * map, char *key, int key_length, int val) -{ - char *pTree = map->data; - dprintf("MPID_nem_ib_cm_map_set,val=%d\n", val); - - if (!pTree) { - MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); - dprintf("pTree was empty\n"); - return; - } - int s1_minus_s2; - while (1) { - int lmin = - key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); - int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree); - s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin); - - if (!s1_minus_s2 && !residual) { - MPID_NEM_IB_MAP_VAL(pTree) = val; - dprintf("found\n"); - return; // same string, same length - } - else if (s1_minus_s2 < 0 || !s1_minus_s2 && residual < 0) { - // psArg is "smaller" OR same substring, psArg is shorter - if (MPID_NEM_IB_MAP_LPTR(pTree) == 0) { - MPID_NEM_IB_MAP_LPTR(pTree) = map->length; // pointer write - /* left child */ - MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); - dprintf("stored as left child\n"); - return; - } - pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child - } - else { - // psArg is "larger" OR same substring, psArg is longer - if (MPID_NEM_IB_MAP_RPTR(pTree) == 0) { - MPID_NEM_IB_MAP_RPTR(pTree) = map->length; // pointer write - /* right child */ - MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); - dprintf("stored as right child\n"); - return; - } - pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child - } - } -} - -int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length, int *val) -{ - int mpi_errno = MPI_SUCCESS; - char *pTree = map->data; - - dprintf("MPID_nem_ib_cm_map_get,key=%s\n", key); - - if (!pTree) { - mpi_errno = -1; - dprintf("pTree is empty\n"); - goto fn_fail; - } - int s1_minus_s2; - while (1) { - int lmin = - key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); - int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree); - s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin); - - if (!s1_minus_s2 && !residual) { - *val = MPID_NEM_IB_MAP_VAL(pTree); - dprintf("value found=%d\n", *val); - goto fn_exit; // same string, same length - } - else if (s1_minus_s2 < 0 || !s1_minus_s2 && residual < 0) { - // psArg is "smaller" OR same substring, psArg is shorter - if (MPID_NEM_IB_MAP_LPTR(pTree) == 0) { - mpi_errno = -1; - dprintf("left is null\n"); - goto fn_fail; - } - pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child - } - else { - // psArg is "larger" OR same substring, psArg is longer - if (MPID_NEM_IB_MAP_RPTR(pTree) == 0) { - mpi_errno = -1; - dprintf("right is null\n"); - goto fn_fail; - } - pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child - } - } - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} -#endif -#endif - #undef FUNCNAME #define FUNCNAME MPID_nem_ib_iSendContig_core #undef FCNAME @@ -171,48 +49,10 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo /* send RDMA-write-to buffer occupancy information */ /* embed SR occupancy information and remember the last one sent */ -#if 0 - MPIDI_CH3_Pkt_t *ch3_hdr = (MPIDI_CH3_Pkt_t *) hdr; -#endif if (MPID_nem_ib_diff16(vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent) > notify_rate) { -#if 0 /* debug, disabling piggy-back */ - switch (ch3_hdr->type) { - case MPIDI_CH3_PKT_EAGER_SEND: - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_EAGER_SEND; - goto common_tail; -#if 0 /* modification of mpid_nem_lmt.c is required */ - case MPIDI_NEM_PKT_LMT_RTS: - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_LMT_RTS; - goto common_tail; -#endif - case MPIDI_CH3_PKT_PUT: - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_PUT; - goto common_tail; - case MPIDI_CH3_PKT_ACCUMULATE: - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_ACCUMULATE; - goto common_tail; - case MPIDI_CH3_PKT_GET: - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_GET; - goto common_tail; - case MPIDI_CH3_PKT_GET_RESP: - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_GET_RESP; - common_tail: - pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; - pkt_netmod.seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; - vc_ib->ibcom->rsr_seq_num_tail_last_sent = vc_ib->ibcom->rsr_seq_num_tail; - prefix = (void *) &pkt_netmod; - sz_prefix = sizeof(MPID_nem_ib_pkt_prefix_t); - break; - default: - prefix = NULL; - sz_prefix = 0; - break; - } -#else prefix = NULL; sz_prefix = 0; -#endif } else { prefix = NULL; @@ -222,28 +62,13 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo s_data = data; s_data_sz = data_sz; -#if 0 - if (hdr && - ((((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_PUT) - || (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_GET_RESP) - || (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_ACCUMULATE))) { -#endif /* If request length is too long, create LMT packet */ if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) + sizeof(MPIDI_CH3_Pkt_t) + data_sz > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; -#if 0 - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_PUT) - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_PUT; - else if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_GET_RESP) - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_GET_RESP; - else if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_ACCUMULATE) - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_ACCUMULATE; -#else pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; -#endif void *write_from_buf = data; @@ -251,11 +76,6 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &max_msg_sz, sizeof(uint32_t)); -#if 0 - /* RMA : Netmod IB supports only smaller size than max_msg_sz. */ - MPIU_Assert(data_sz <= max_msg_sz); -#endif - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t)); sreq->ch.s_cookie = s_cookie_buf; @@ -289,9 +109,6 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo int incomplete; MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone } -#if 0 - } -#endif /* packet handlers including MPIDI_CH3_PktHandler_EagerSend and MPID_nem_handle_pkt assume this */ hdr_sz = sizeof(MPIDI_CH3_Pkt_t); @@ -344,14 +161,7 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo MPID_nem_ib_ncqe += 1; //dprintf("isendcontig_core,ncqe=%d\n", MPID_nem_ib_ncqe); dprintf("isendcontig_core,isend,kind=%d,msg_type=%d,copied=%d\n", sreq->kind, msg_type, copied); /*suspicious lines,(if1,on,on,off,if0) works */ -#if 0 -#define MPID_NEM_IB_TLBPREF_SEND 20 //20 - int tlb_pref_ahd = 4096 * MPID_NEM_IB_TLBPREF_SEND; - __asm__ __volatile__ - ("movq %0, %%rsi;" - "movq 0(%%rsi), %%rax;"::"r"((uint64_t) data + tlb_pref_ahd):"%rsi", "%rax"); -#endif -#if 1 + #ifdef __MIC__ __asm__ __volatile__ ("movq %0, %%rsi;" @@ -367,7 +177,6 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo "prefetchnta 0x80(%%rsi);" "prefetchnta 0xc0(%%rsi);"::"r"((uint64_t) data + 4 * data_sz):"%rsi"); #endif -#endif MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "ib_send, fd=%d", vc_ib->sc->fd)); vc_ib->pending_sends += 1; @@ -383,46 +192,8 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo * so we need to perform * progress_send for all of VCs using nces in ib_poll * (if we have drain-sendq in ib_poll, this isn't needed. */ -#if 0 /* debug,disabling fast-dec-cc when copied */ - if (copied && !sreq->dev.OnDataAvail) { /* skip poll scq */ - int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); - - (VC_FIELD(sreq->ch.vc, pending_sends)) -= 1; - - /* as in the template */ - reqFn = sreq->dev.OnDataAvail; - if (!reqFn) { - /* MPID_Request_release is called in - * MPI_Wait (in src/mpi/pt2pt/wait.c) - * MPIR_Wait_impl (in src/mpi/pt2pt/wait.c) - * MPIR_Request_complete (in /src/mpi/pt2pt/mpir_request.c) */ - int incomplete; - MPIDI_CH3U_Request_decrement_cc(sreq, &incomplete); - if (!incomplete) { - MPIDI_CH3_Progress_signal_completion(); - } - //dprintf("isendcontig_core,cc_ptr=%d\n", *(sreq->cc_ptr)); - dprintf("sendcontig_core,copied,complete,req=%p,cc incremented to %d,ref_count=%d\n", - sreq, MPIDI_CH3I_progress_completion_count.v, sreq->ref_count); - MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); - } - else { - MPIDI_VC_t *vc = sreq->ch.vc; - int complete = 0; - mpi_errno = reqFn(vc, sreq, &complete); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - /* not-completed case is not implemented */ - MPIU_Assert(complete == TRUE); - MPIU_Assert(0); /* decrement ref_count and free sreq causes problem */ - } - } - else { - MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ - } -#else + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ -#endif #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE //dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); @@ -464,9 +235,6 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; -#if 0 - int ibcom_errno; -#endif MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ISENDCONTIG); @@ -509,42 +277,15 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, } } -#if 0 - /* aggressively perform drain_scq */ - /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_ib->ibcom->ncom >= - /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_ib_ncqe >= - /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { - //printf("isendcontig,kick drain_scq\n"); - ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); - } -#endif /* set it for drain_scq */ MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_EAGER_MSG); - - #ifdef MPID_NEM_IB_ONDEMAND if (vc_ib->connection_state != MPID_NEM_IB_CM_ESTABLISHED) { goto enqueue; } #endif -#if 0 - /* anticipating received message releases RDMA-write-to buffer or IB command-queue entry */ - /* Unexpected state MPIDI_VC_STATE_CLOSED in vc 0xf1fed0 (expecting MPIDI_VC_STATE_ACTIVE) - * Assertion failed in file src/mpid/ch3/src/ch3u_handle_connection.c at line 326: vc->state == MPIDI_VC_STATE_ACTIVE */ - if (vc->state == MPIDI_VC_STATE_ACTIVE && - MPID_nem_ib_tsc_poll - MPID_nem_ib_rdtsc() > MPID_NEM_IB_POLL_PERIOD_SEND) { - mpi_errno = MPID_nem_ib_poll(0); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - } -#endif - dprintf ("isendcontig,%d->%d,req=%p,type=%d,subtype=%d,data_sz=%ld,ldiff=%d(%d-%d),rdiff=%d(%d-%d)\n", MPID_nem_ib_myrank, vc->pg_rank, sreq, ((MPIDI_CH3_Pkt_t *) hdr)->type, @@ -638,39 +379,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, * rank-0 stops draining command queue. */ dprintf("isendcontig,enqueuing,type=%d,\n", ((MPIDI_CH3_Pkt_t *) hdr)->type); -#if 0 - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_NETMOD) { - switch (((MPID_nem_pkt_netmod_t *) hdr)->subtype) { - case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM: - dprintf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_ib_myrank, vc->pg_rank, - MPID_nem_ib_ncqe); - break; - default: - break; - } - } - else { - switch (((MPIDI_CH3_Pkt_t *) hdr)->type) { - case MPIDI_CH3_PKT_ACCUMULATE: - dprintf("enqueuing ACCUMULATE\n"); - break; - case MPIDI_CH3_PKT_GET_RESP: - dprintf("enqueuing GET_RESP\n"); - break; - case MPIDI_CH3_PKT_GET: - dprintf("enqueuing GET\n"); - break; - case MPIDI_CH3_PKT_PUT: - dprintf("enqueuing PUT\n"); - break; - case MPIDI_NEM_PKT_LMT_DONE: - dprintf("isendcontig,enqueue,DONE\n"); - break; - default: - break; - } - } -#endif + if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_NETMOD && ((MPID_nem_pkt_netmod_t *) hdr)->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { dprintf("isendcontig,REPLY_SEQ_NUM,enqueue_at_head\n"); @@ -700,12 +409,6 @@ int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_s { MPID_Request *sreq = NULL; int mpi_errno = MPI_SUCCESS; -#if 0 - int ibcom_errno; - MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - int sseq_num; -#endif - //uint64_t tscs, tsce; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ISTARTCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_ISTARTCONTIGMSG); @@ -730,20 +433,6 @@ int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_s sreq->dev.OnDataAvail = 0; //tsce = MPID_nem_ib_rdtsc(); printf("rc,%ld\n", tsce - tscs); // 124.15 cycles -#if 0 - if (hdr) { - - MPIDI_CH3_Pkt_t *pkt = (MPIDI_CH3_Pkt_t *) hdr; - MPIDI_CH3_Pkt_close_t *close_pkt = &pkt->close; - dprintf("isend(istartcontig),%d->%d,seq_num=%d,type=%d,ack=%d\n", MPID_nem_ib_myrank, - vc->pg_rank, vc_ib->ibcom->sseq_num, close_pkt->type, close_pkt->ack); - } - else { - dprintf("isend(istartcontig),%d->%d,seq_num=%d\n", MPID_nem_ib_myrank, vc->pg_rank, - vc_ib->ibcom->sseq_num); - } -#endif - mpi_errno = MPID_nem_ib_iSendContig(vc, sreq, hdr, hdr_sz, data, data_sz); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -795,28 +484,13 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, data = (void *)REQ_FIELD(sreq, lmt_pack_buf); data_sz = last; -#if 0 - if (hdr && - ((((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_PUT) - || (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_GET_RESP) - || (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_ACCUMULATE))) { -#endif /* If request length is too long, create LMT packet */ if ( MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type) + sizeof(MPIDI_CH3_Pkt_t) + sreq->dev.segment_size - sreq->dev.segment_first > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) { pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; -#if 0 - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_PUT) - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_PUT; - else if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_GET_RESP) - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_GET_RESP; - else if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_ACCUMULATE) - pkt_netmod.subtype = MPIDI_NEM_IB_PKT_ACCUMULATE; -#else pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS; -#endif void *write_from_buf = REQ_FIELD(sreq, lmt_pack_buf); @@ -824,11 +498,6 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &max_msg_sz, sizeof(uint32_t)); -#if 0 - /* RMA : Netmod IB supports only smaller size than max_msg_sz. */ - MPIU_Assert(data_sz <= max_msg_sz); -#endif - MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t)); sreq->ch.s_cookie = s_cookie_buf; @@ -862,9 +531,6 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, int incomplete; MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone } -#if 0 - } -#endif /* packet handlers assume this */ hdr_sz = sizeof(MPIDI_CH3_Pkt_t); @@ -900,49 +566,9 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vc_ib->pending_sends += 1; sreq->ch.vc = vc; /* used in poll */ -#if 0 /* see contig */ - if (copied) { /* skip poll scq */ - int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); - - (VC_FIELD(sreq->ch.vc, pending_sends)) -= 1; - - /* as in the template */ - reqFn = sreq->dev.OnDataAvail; - if (!reqFn) { - /* MPID_Request_release is called in - * MPI_Wait (in src/mpi/pt2pt/wait.c) - * MPIR_Wait_impl (in src/mpi/pt2pt/wait.c) - * MPIR_Request_complete (in /src/mpi/pt2pt/mpir_request.c) */ - int incomplete; - MPIDI_CH3U_Request_decrement_cc(sreq, &incomplete); - if (!incomplete) { - MPIDI_CH3_Progress_signal_completion(); - } - //dprintf("isendcontig_core,cc_ptr=%d\n", *(sreq->cc_ptr)); - dprintf("sendcontig_core,complete,req=%p,cc incremented to %d\n", sreq, - MPIDI_CH3I_progress_completion_count.v); - MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); - } - else { - MPIDI_VC_t *vc = sreq->ch.vc; - int complete = 0; - mpi_errno = reqFn(vc, sreq, &complete); - if (mpi_errno) - MPIU_ERR_POP(mpi_errno); - /* not-completed case is not implemented */ - MPIU_Assert(complete == TRUE); - MPIU_Assert(0); /* decrement ref_count and free sreq causes problem */ - } - } - else { - MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ - } -#else MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ -#endif #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE -#if 1 //dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); int *notify_rstate; ibcom_errno = @@ -966,7 +592,6 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, } //dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif -#endif fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG_CORE); return mpi_errno; @@ -983,10 +608,6 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; -#if 0 - int ibcom_errno; - MPIDI_msg_sz_t last; -#endif MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG); @@ -1029,18 +650,6 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, dprintf("sendnoncontig,%d->%d,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); -#if 0 - /* aggressively perform drain_scq */ - /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_ib->ibcom->ncom >= - /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_ib_ncqe >= - /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { - //printf("isendcontig,kick drain_scq\n"); - ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); - } -#endif /* set it for drain_scq */ MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_EAGER_MSG); @@ -1097,9 +706,6 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; -#if 0 - int ibcom_errno; -#endif MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); MPID_Request *sreq, *prev_sreq; int req_type, msg_type; @@ -1126,19 +732,6 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) if (sreq) { prev_sreq = NULL; do { -#if 0 - /* aggressively perform drain_scq */ - /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_ib->ibcom->ncom >= - /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_ib_ncqe >= - /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { - dprintf("send_progress,kick drain_scq\n"); - ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_drain_scq"); - } -#endif req_type = MPIDI_Request_get_type(sreq); msg_type = MPIDI_Request_get_msg_type(sreq); @@ -1257,36 +850,14 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) /* send current rsr_seq_num_tail because message from target to initiator * might have happened while being queued */ case MPIDI_NEM_PKT_LMT_RTS:{ -#if 0 - MPID_nem_ib_lmt_cookie_t *s_cookie_buf = - (MPID_nem_ib_lmt_cookie_t *) sreq->dev.iov[1].MPID_IOV_BUF; -#endif dprintf("send_progress,MPIDI_NEM_PKT_LMT_RTS,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); -#if 0 /* moving to packet header */ - /* embed RDMA-write-to buffer occupancy information */ - s_cookie_buf->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; - /* remember the last one sent */ - vc_ib->ibcom->rsr_seq_num_tail_last_sent = - vc_ib->ibcom->rsr_seq_num_tail; -#endif break; } case MPIDI_NEM_PKT_LMT_CTS:{ -#if 0 - MPID_nem_ib_lmt_cookie_t *s_cookie_buf = - (MPID_nem_ib_lmt_cookie_t *) sreq->dev.iov[1].MPID_IOV_BUF; -#endif dprintf("send_progress,MPIDI_NEM_PKT_LMT_CTS,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); -#if 0 /* moving to packet header */ - /* embed RDMA-write-to buffer occupancy information */ - s_cookie_buf->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; - /* remember the last one sent */ - vc_ib->ibcom->rsr_seq_num_tail_last_sent = - vc_ib->ibcom->rsr_seq_num_tail; -#endif break; } @@ -1298,19 +869,6 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc) /* send current rsr_seq_num_tail because message from target to initiator * might have happened while being queued */ case MPIDI_NEM_IB_PKT_LMT_GET_DONE:{ -#if 0 - MPID_nem_ib_pkt_lmt_get_done_t *_done_pkt = - (MPID_nem_ib_pkt_lmt_get_done_t *) sreq->dev.iov[0]. - MPID_IOV_BUF; - dprintf - ("send_progress,MPIDI_NEM_IB_PKT_LMT_GET_DONE,rsr_seq_num_tail=%d\n", - vc_ib->ibcom->rsr_seq_num_tail); - /* embed SR occupancy information */ - _done_pkt->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; - /* remember the last one sent */ - vc_ib->ibcom->rsr_seq_num_tail_last_sent = - vc_ib->ibcom->rsr_seq_num_tail; -#endif break; } case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:{ @@ -1452,12 +1010,6 @@ int MPID_nem_ib_cm_progress() /* This comparison is OK if the diff is within 63-bit range */ if (MPID_nem_ib_diff63(MPID_nem_ib_progress_engine_vt, sreq->retry_decided) < sreq->retry_backoff) { -#if 0 - dprintf("cm_progress,vt=%ld,retry_decided=%ld,diff=%ld,backoff=%ld\n", - MPID_nem_ib_progress_engine_vt, sreq->retry_decided, - MPID_nem_ib_diff63(MPID_nem_ib_progress_engine_vt, sreq->retry_decided), - sreq->retry_backoff); -#endif goto next; } dprintf @@ -1488,7 +1040,6 @@ int MPID_nem_ib_cm_progress() break; case MPID_NEM_IB_CM_SYN: if (is_conn_established(sreq->responder_rank)) { -#if 1 /* Explicitly release CAS word because * ConnectX-3 doesn't support safe CAS with PCI device and CPU */ MPID_nem_ib_cm_cas_release(MPID_nem_ib_conns[sreq->responder_rank].vc); @@ -1497,36 +1048,6 @@ int MPID_nem_ib_cm_progress() sreq->ibcom->outstanding_connection_tx); is_established = 1; break; -#else - dprintf("cm_progress,syn,switching to cas_release,%d->%d\n", - MPID_nem_ib_myrank, sreq->responder_rank); - /* Connection was established while SYN command was enqueued. - * So we replace SYN with CAS_RELEASE, and send. */ - - /* override req->type */ - ((MPID_nem_ib_cm_cmd_syn_t *) & sreq->cmd)->type = MPID_NEM_IB_CM_CAS_RELEASE2; - ((MPID_nem_ib_cm_cmd_syn_t *) & sreq->cmd)->initiator_rank = MPID_nem_ib_myrank; - - /* Initiator does not receive SYNACK and ACK2, so we decrement incoming counter here. */ - sreq->ibcom->incoming_connection_tx -= 2; - - shadow = - (MPID_nem_ib_cm_cmd_shadow_t *) - MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t)); - - /* override req->state */ - shadow->type = sreq->state = MPID_NEM_IB_CM_CAS_RELEASE2; - shadow->req = sreq; - dprintf("shadow=%p,shadow->req=%p\n", shadow, shadow->req); - mpi_errno = - MPID_nem_ib_cm_cmd_core(sreq->responder_rank, shadow, - (void *) (&sreq->cmd), - sizeof(MPID_nem_ib_cm_cmd_synack_t), 1 /* syn:1 */ , - 0); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_ib_cm_send_core"); -#endif - break; } /* The initiator acqurire slot for the responder when sending syn */ @@ -2225,9 +1746,7 @@ int MPID_nem_ib_ringbuf_ask_cas(MPIDI_VC_t * vc, MPID_nem_ib_ringbuf_req_t * req /* Ring-buffer is full */ /* Let the guard down so that this ask-fetch can be issued in ringbuf_progress */ -#if 0 /*debug */ - VC_FIELD(vc, ibcom->ask_guard) = 0; -#endif + /* Retry from fetch */ /* Schedule retry */ ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/netmod/ib/ib_finalize.c | 15 +- src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c | 401 +------- src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h | 8 +- src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h | 81 +- src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c | 141 +--- src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c | 127 +-- .../ch3/channels/nemesis/netmod/ib/ib_malloc.c | 12 +- src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c | 1100 +++----------------- .../ch3/channels/nemesis/netmod/ib/ib_reg_mr.c | 112 +-- src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c | 773 +++----------- 10 files changed, 438 insertions(+), 2332 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-41-g32e24c1
by noreply＠mpich.org 07 Apr '15

07 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 32e24c1db7163dfd7f13f8bba6c9321cc53c6972 (commit) from 05fe55b2d543f6f70a839faacb94aa2cb851720a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/32e24c1db7163dfd7f13f8bba6c9321cc… commit 32e24c1db7163dfd7f13f8bba6c9321cc53c6972 Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Tue Mar 31 15:45:34 2015 -0500 portals4: improvements to VC close Use the VC private area to track outstanding send operations. This way, when a VC close packet comes in, we wait until all remaining operations are complete before closing locally. This allows for a simpler netmod finalize function where we are sure the network is safe to shutdown. Signed-off-by: Antonio J. Pena <apenya(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c index 27cbc02..47bc252 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c @@ -19,7 +19,6 @@ static char recvbufs[NUM_RECV_BUFS * PTL_MAX_EAGER]; static ptl_me_t mes[NUM_RECV_BUFS]; static ptl_handle_me_t me_handles[NUM_RECV_BUFS]; -static unsigned long long put_cnt = 0; /* required to not finalizing too early */ #undef FUNCNAME #define FUNCNAME MPID_nem_ptl_nm_init @@ -76,8 +75,6 @@ int MPID_nem_ptl_nm_finalize(void) MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_NM_FINALIZE); - while (put_cnt) MPID_nem_ptl_poll(1); /* Wait for puts to finish */ - for (i = 0; i < NUM_RECV_BUFS; ++i) { ret = PtlMEUnlink(me_handles[i]); MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", @@ -176,6 +173,7 @@ static inline int send_pkt(MPIDI_VC_t *vc, void *hdr_p, void *data_p, MPIDI_msg_ } SENDBUF(sreq) = sendbuf; + sreq->ch.vc = vc; REQ_PTL(sreq)->event_handler = MPID_nem_ptl_nm_ctl_event_handler; ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)sendbuf, sendbuf_sz, PTL_NO_ACK_REQ, @@ -185,7 +183,8 @@ static inline int send_pkt(MPIDI_VC_t *vc, void *hdr_p, void *data_p, MPIDI_msg_ MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlPut(size=%lu id=(%#x,%#x) pt=%#x)", sendbuf_sz, vc_ptl->id.phys.nid, vc_ptl->id.phys.pid, vc_ptl->ptc)); - ++put_cnt; + + vc_ptl->num_queued_sends++; fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_SEND_PKT); @@ -238,6 +237,7 @@ static int send_noncontig_pkt(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr_p) } SENDBUF(sreq) = sendbuf; + sreq->ch.vc = vc; REQ_PTL(sreq)->event_handler = MPID_nem_ptl_nm_ctl_event_handler; ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)sendbuf, sendbuf_sz, PTL_NO_ACK_REQ, @@ -247,7 +247,8 @@ static int send_noncontig_pkt(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr_p) MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlPut(size=%lu id=(%#x,%#x) pt=%#x)", sendbuf_sz, vc_ptl->id.phys.nid, vc_ptl->id.phys.pid, vc_ptl->ptc)); - ++put_cnt; + + vc_ptl->num_queued_sends++; fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_SEND_NONCONTIG_PKT); @@ -340,6 +341,9 @@ int MPID_nem_ptl_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPID static inline void on_data_avail(MPID_Request * req) { int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); + MPIDI_VC_t *vc = req->ch.vc; + MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc); + MPIDI_STATE_DECL(MPID_STATE_ON_DATA_AVAIL); MPIDI_FUNC_ENTER(MPID_STATE_ON_DATA_AVAIL); @@ -351,12 +355,14 @@ static inline void on_data_avail(MPID_Request * req) } else { int complete; - MPIDI_VC_t *vc = req->ch.vc; reqFn(vc, req, &complete); MPIU_Assert(complete == TRUE); } - --put_cnt; + vc_ptl->num_queued_sends--; + + if (vc->state == MPIDI_VC_STATE_CLOSED && vc_ptl->num_queued_sends == 0) + MPID_nem_ptl_vc_terminated(vc); MPIDI_FUNC_EXIT(MPID_STATE_ON_DATA_AVAIL); } diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c index d1ad140..f742ae8 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c @@ -64,6 +64,9 @@ static int handler_send(const ptl_event_t *e) { int mpi_errno = MPI_SUCCESS; MPID_Request *const sreq = e->user_ptr; + MPIDI_VC_t *vc = sreq->ch.vc; + MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc); + int i, ret, incomplete; MPIDI_STATE_DECL(MPID_STATE_HANDLER_SEND); @@ -87,6 +90,10 @@ static int handler_send(const ptl_event_t *e) MPIU_Free(REQ_PTL(sreq)->get_me_p); MPIDI_CH3U_Request_complete(sreq); + vc_ptl->num_queued_sends--; + + if (vc->state == MPIDI_VC_STATE_CLOSED && vc_ptl->num_queued_sends == 0) + MPID_nem_ptl_vc_terminated(vc); } else { MPIDI_CH3U_Request_decrement_cc(sreq, &incomplete); } @@ -127,6 +134,8 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void * sreq->dev.match.parts.rank = dest; sreq->dev.match.parts.tag = tag; sreq->dev.match.parts.context_id = comm->context_id + context_offset; + sreq->ch.vc = vc; + vc_ptl->num_queued_sends++; if (!vc_ptl->id_initialized) { mpi_errno = MPID_nem_ptl_init_id(vc); ----------------------------------------------------------------------- Summary of changes: .../ch3/channels/nemesis/netmod/portals4/ptl_nm.c | 20 +++++++++++++------- .../channels/nemesis/netmod/portals4/ptl_send.c | 9 +++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-40-g05fe55b
by noreply＠mpich.org 07 Apr '15

07 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 05fe55b2d543f6f70a839faacb94aa2cb851720a (commit) via b47d95f7a04a0fdbb82b4b5bf4a64ddb5c4b48aa (commit) from c6777161b2324bd5daaf555b99362d792368a797 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/05fe55b2d543f6f70a839faacb94aa2cb… commit 05fe55b2d543f6f70a839faacb94aa2cb851720a Author: Antonio J. Pena <apenya(a)mcs.anl.gov> Date: Tue Apr 7 14:08:36 2015 -0500 Fix win_shared_create* tests The tests were modifying local buffers without locking them after window creation, causing potential race conditions. I've moved the buffer initialization to be performed before the global window is created. These tests were failing due to incorrect results in Jenkins whith async enabled. Signed-off-by: Xin Zhao <xinzhao3(a)illinois.edu> diff --git a/test/mpi/rma/win_shared_create.c b/test/mpi/rma/win_shared_create.c index eec22d6..0042184 100644 --- a/test/mpi/rma/win_shared_create.c +++ b/test/mpi/rma/win_shared_create.c @@ -74,15 +74,15 @@ int main(int argc, char **argv) create_info = MPI_INFO_NULL; #endif - MPI_Win_create(my_base, sizeof(int) * ELEM_PER_PROC, sizeof(int), create_info, MPI_COMM_WORLD, - &win); - /* Reset data */ for (i = 0; i < ELEM_PER_PROC; i++) { my_base[i] = 0; local_buf[i] = i + 1; } + MPI_Win_create(my_base, sizeof(int) * ELEM_PER_PROC, sizeof(int), create_info, MPI_COMM_WORLD, + &win); + /* Do RMA through global window, then check value through shared window */ MPI_Win_lock_all(0, win); MPI_Win_lock_all(0, shm_win); http://git.mpich.org/mpich.git/commitdiff/b47d95f7a04a0fdbb82b4b5bf4a64ddb5… commit b47d95f7a04a0fdbb82b4b5bf4a64ddb5c4b48aa Author: Antonio J. Pena <apenya(a)mcs.anl.gov> Date: Tue Apr 7 14:07:14 2015 -0500 Fix unsafe datatype release in several RMA tests The datatypes shouldn't be released until we make sure that there are no more remote operations using that datatype. I've changed several tests to release the datatype after a barrier. To avoid introducing a barrier in every iteration, and aiming to stress out a little more, I've restructured the tests so that the datatypes are not created and freed every iteration. This was causing intermittent segfaults mainly with async enabled. Signed-off-by: Xin Zhao <xinzhao3(a)illinois.edu> diff --git a/test/mpi/rma/put_base.c b/test/mpi/rma/put_base.c index ba95a1c..a88f1eb 100644 --- a/test/mpi/rma/put_base.c +++ b/test/mpi/rma/put_base.c @@ -36,6 +36,12 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; + MPI_Aint idx_loc[SUB_YDIM]; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; + void *base_ptr; + MPI_Aint base_int; MTest_Init(&argc, &argv); @@ -58,42 +64,35 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided put operations */ - - for (i = 0; i < ITERATIONS; i++) { - MPI_Aint idx_loc[SUB_YDIM]; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; + base_ptr = dst_buf; - void *base_ptr = dst_buf; - MPI_Aint base_int; + MPI_Get_address(base_ptr, &base_int); - MPI_Get_address(base_ptr, &base_int); - - for (j = 0; j < SUB_YDIM; j++) { + for (j = 0; j < SUB_YDIM; j++) { MPI_Get_address(&src_buf[j*XDIM], &idx_loc[j]); idx_loc[j] = idx_loc[j] - base_int; idx_rem[j] = j*XDIM*sizeof(double); blk_len[j] = SUB_XDIM*sizeof(double); - } + } - MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); - MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); + MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); + MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided put operations */ + for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(base_ptr, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/put_bottom.c b/test/mpi/rma/put_bottom.c index 6634ea0..0bd0b7f 100644 --- a/test/mpi/rma/put_bottom.c +++ b/test/mpi/rma/put_bottom.c @@ -34,6 +34,10 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; + MPI_Aint idx_loc[SUB_YDIM]; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -54,36 +58,30 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided put operations */ - - for (i = 0; i < ITERATIONS; i++) { - MPI_Aint idx_loc[SUB_YDIM]; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - - for (j = 0; j < SUB_YDIM; j++) { + for (j = 0; j < SUB_YDIM; j++) { MPI_Get_address(&src_buf[j*XDIM], &idx_loc[j]); idx_rem[j] = j*XDIM*sizeof(double); blk_len[j] = SUB_XDIM*sizeof(double); - } + } - MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); - MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); + MPI_Type_create_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_BYTE, &src_type); + MPI_Type_create_indexed_block(SUB_YDIM, SUB_XDIM*sizeof(double), idx_rem, MPI_BYTE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided put operations */ + for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(MPI_BOTTOM, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_acc_indexed.c b/test/mpi/rma/strided_acc_indexed.c index ac54f52..45254c2 100644 --- a/test/mpi/rma/strided_acc_indexed.c +++ b/test/mpi/rma/strided_acc_indexed.c @@ -32,6 +32,10 @@ int main(int argc, char **argv) { int itr, i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf; MPI_Win buf_win; + MPI_Aint idx_loc[SUB_YDIM]; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -51,30 +55,24 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided accumulate operations */ - - for (itr = 0; itr < ITERATIONS; itr++) { - MPI_Aint idx_loc[SUB_YDIM]; - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - - for (i = 0; i < SUB_YDIM; i++) { + for (i = 0; i < SUB_YDIM; i++) { MPI_Get_address(&src_buf[i*XDIM], &idx_loc[i]); idx_rem[i] = i*XDIM; blk_len[i] = SUB_XDIM; - } + } #ifdef ABSOLUTE - MPI_Type_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_DOUBLE, &src_type); + MPI_Type_hindexed(SUB_YDIM, blk_len, idx_loc, MPI_DOUBLE, &src_type); #else - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); #endif - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided accumulate operations */ + for (itr = 0; itr < ITERATIONS; itr++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); #ifdef ABSOLUTE @@ -84,13 +82,13 @@ int main(int argc, char **argv) { #endif MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_acc_subarray.c b/test/mpi/rma/strided_acc_subarray.c index c8f850c..385c928 100644 --- a/test/mpi/rma/strided_acc_subarray.c +++ b/test/mpi/rma/strided_acc_subarray.c @@ -32,6 +32,14 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf; MPI_Win buf_win; + int ndims = 2; + int src_arr_sizes[2] = { XDIM, YDIM }; + int src_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; + int src_arr_starts[2] = { 0, 0 }; + int dst_arr_sizes[2] = { XDIM, YDIM }; + int dst_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; + int dst_arr_starts[2] = { 0, 0 }; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -51,39 +59,27 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided accumulate operations */ - - for (i = 0; i < ITERATIONS; i++) { - int ndims = 2; - int src_arr_sizes[2] = { XDIM, YDIM }; - int src_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; - int src_arr_starts[2] = { 0, 0 }; - int dst_arr_sizes[2] = { XDIM, YDIM }; - int dst_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM }; - int dst_arr_starts[2] = { 0, 0 }; - MPI_Datatype src_type, dst_type; - - MPI_Type_create_subarray(ndims, src_arr_sizes, src_arr_subsizes, src_arr_starts, + MPI_Type_create_subarray(ndims, src_arr_sizes, src_arr_subsizes, src_arr_starts, MPI_ORDER_C, MPI_DOUBLE, &src_type); - MPI_Type_create_subarray(ndims, dst_arr_sizes, dst_arr_subsizes, dst_arr_starts, + MPI_Type_create_subarray(ndims, dst_arr_sizes, dst_arr_subsizes, dst_arr_starts, MPI_ORDER_C, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided accumulate operations */ + for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); - MPI_Accumulate(src_buf, 1, src_type, peer, 0, 1, dst_type, MPI_SUM, buf_win); - MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_get_indexed.c b/test/mpi/rma/strided_get_indexed.c index 3a98d29..37effd5 100644 --- a/test/mpi/rma/strided_get_indexed.c +++ b/test/mpi/rma/strided_get_indexed.c @@ -78,11 +78,11 @@ int main(int argc, char **argv) { MPI_Win_unlock(peer, buf_win); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&loc_type); MPI_Type_free(&rem_type); - MPI_Barrier(MPI_COMM_WORLD); - /* Verify that the results are correct */ errors = 0; diff --git a/test/mpi/rma/strided_getacc_indexed.c b/test/mpi/rma/strided_getacc_indexed.c index e3293a1..2d52e5b 100644 --- a/test/mpi/rma/strided_getacc_indexed.c +++ b/test/mpi/rma/strided_getacc_indexed.c @@ -32,6 +32,9 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -52,24 +55,19 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided accumulate operations */ - - for (i = 0; i < ITERATIONS; i++) { - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - - for (j = 0; j < SUB_YDIM; j++) { + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided accumulate operations */ + for (i = 0; i < ITERATIONS; i++) { /* PUT */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, @@ -81,13 +79,13 @@ int main(int argc, char **argv) { MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, 1, dst_type, MPI_NO_OP, buf_win); MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_getacc_indexed_shared.c b/test/mpi/rma/strided_getacc_indexed_shared.c index e1c8169..a67fdff 100644 --- a/test/mpi/rma/strided_getacc_indexed_shared.c +++ b/test/mpi/rma/strided_getacc_indexed_shared.c @@ -34,6 +34,9 @@ int main(int argc, char **argv) { double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; MPI_Comm shr_comm; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -62,24 +65,19 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided accumulate operations */ - - for (i = 0; i < ITERATIONS; i++) { - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - - for (j = 0; j < SUB_YDIM; j++) { + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided accumulate operations */ + for (i = 0; i < ITERATIONS; i++) { /* PUT */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, @@ -91,13 +89,13 @@ int main(int argc, char **argv) { MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, 1, dst_type, MPI_NO_OP, buf_win); MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_putget_indexed.c b/test/mpi/rma/strided_putget_indexed.c index 09f17ae..677e196 100644 --- a/test/mpi/rma/strided_putget_indexed.c +++ b/test/mpi/rma/strided_putget_indexed.c @@ -32,6 +32,9 @@ int main(int argc, char **argv) { int i, j, rank, nranks, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -52,24 +55,19 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided accumulate operations */ - - for (i = 0; i < ITERATIONS; i++) { - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - - for (j = 0; j < SUB_YDIM; j++) { + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided accumulate operations */ + for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(src_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); @@ -77,13 +75,13 @@ int main(int argc, char **argv) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get(dst_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); diff --git a/test/mpi/rma/strided_putget_indexed_shared.c b/test/mpi/rma/strided_putget_indexed_shared.c index 1e65f2a..2d4f02e 100644 --- a/test/mpi/rma/strided_putget_indexed_shared.c +++ b/test/mpi/rma/strided_putget_indexed_shared.c @@ -34,6 +34,9 @@ int main(int argc, char **argv) { double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; MPI_Comm shr_comm; + int idx_rem[SUB_YDIM]; + int blk_len[SUB_YDIM]; + MPI_Datatype src_type, dst_type; MTest_Init(&argc, &argv); @@ -62,24 +65,19 @@ int main(int argc, char **argv) { peer = (rank+1) % nranks; - /* Perform ITERATIONS strided accumulate operations */ - - for (i = 0; i < ITERATIONS; i++) { - int idx_rem[SUB_YDIM]; - int blk_len[SUB_YDIM]; - MPI_Datatype src_type, dst_type; - - for (j = 0; j < SUB_YDIM; j++) { + for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; - } + } - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); - MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); + MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); - MPI_Type_commit(&src_type); - MPI_Type_commit(&dst_type); + MPI_Type_commit(&src_type); + MPI_Type_commit(&dst_type); + /* Perform ITERATIONS strided accumulate operations */ + for (i = 0; i < ITERATIONS; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Put(src_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); @@ -87,13 +85,13 @@ int main(int argc, char **argv) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get(dst_buf, 1, src_type, peer, 0, 1, dst_type, buf_win); MPI_Win_unlock(peer, buf_win); - - MPI_Type_free(&src_type); - MPI_Type_free(&dst_type); } MPI_Barrier(shr_comm); + MPI_Type_free(&src_type); + MPI_Type_free(&dst_type); + /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); ----------------------------------------------------------------------- Summary of changes: test/mpi/rma/put_base.c | 39 ++++++++++++------------- test/mpi/rma/put_bottom.c | 32 ++++++++++----------- test/mpi/rma/strided_acc_indexed.c | 34 ++++++++++------------ test/mpi/rma/strided_acc_subarray.c | 38 +++++++++++-------------- test/mpi/rma/strided_get_indexed.c | 4 +- test/mpi/rma/strided_getacc_indexed.c | 30 +++++++++---------- test/mpi/rma/strided_getacc_indexed_shared.c | 30 +++++++++---------- test/mpi/rma/strided_putget_indexed.c | 30 +++++++++---------- test/mpi/rma/strided_putget_indexed_shared.c | 30 +++++++++---------- test/mpi/rma/win_shared_create.c | 6 ++-- 10 files changed, 128 insertions(+), 145 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-38-gc677716
by noreply＠mpich.org 06 Apr '15

06 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via c6777161b2324bd5daaf555b99362d792368a797 (commit) from 36ac1a9da2f1ee4d8204d4a93952d71273d96ae3 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/c6777161b2324bd5daaf555b99362d792… commit c6777161b2324bd5daaf555b99362d792368a797 Author: Sameh Sharkawi <sssharka(a)us.ibm.com> Date: Mon Apr 6 13:54:04 2015 -0400 PAMID: Fix CUDA-aware NULL buffer assert (ibm) D203212 Signed-off-by: Coffman <pkcoff(a)bldlnx65.pok.stglabs.ibm.com> Signed-off-by: Sameh Sharkawi <sssharka(a)us.ibm.com> diff --git a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c index 0987cfd..ec61b77 100644 --- a/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c +++ b/src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c @@ -382,6 +382,8 @@ MPIDI_SendMsg_process_userdefined_dt(MPID_Request * sreq, #if CUDA_AWARE_SUPPORT if(!MPIDI_Process.cuda_aware_support_on || !on_device) buf = sreq->mpid.userbuf; +#else + buf = sreq->mpid.userbuf; #endif MPID_assert(buf != NULL); ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/src/pt2pt/mpidi_sendmsg.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.2b1-37-g36ac1a9
by noreply＠mpich.org 03 Apr '15

03 Apr '15

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 36ac1a9da2f1ee4d8204d4a93952d71273d96ae3 (commit) via 5c4f21d038c2975d968af232447b25aaf6d1442d (commit) via 18516bafa3a3ee7a9e0343ad1bda9876fa52a49a (commit) via 41ab3461e8200920c0fb9a65f8e1921bb428fe87 (commit) via 398aeb4fa597ab55aff917066583e5e665631037 (commit) via e6849ac4f0a85a58afefc7a0ee9643dbf225852f (commit) via 147329674e7989e29c441efa73ba2c4462db5683 (commit) via 0e4dcc43f3ef0be5fb035ceef310aeaf71e19df7 (commit) from c6c0d6f6e67fcf8fb213044f551c6a577ae05dd6 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/36ac1a9da2f1ee4d8204d4a93952d7127… commit 36ac1a9da2f1ee4d8204d4a93952d71273d96ae3 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Fri Apr 3 10:16:21 2015 -0500 move window creation out of common code Instead of creating window at open time (depending on hints), let's deferr the window creation until we need it. Signed-off-by: Paul Coffman <pkcoff(a)us.ibm.com> diff --git a/src/mpi/romio/adio/common/ad_close.c b/src/mpi/romio/adio/common/ad_close.c index ff0049e..01cf1ba 100644 --- a/src/mpi/romio/adio/common/ad_close.c +++ b/src/mpi/romio/adio/common/ad_close.c @@ -7,9 +7,6 @@ #include "adio.h" #include "adio_extern.h" -extern int gpfsmpio_aggmethod; -extern int gpfsmpio_onesided_no_rmw; - #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -117,15 +114,7 @@ void ADIO_Close(ADIO_File fd, int *error_code) MPI_Info_free(&(fd->info)); if (fd->io_buf != NULL) ADIOI_Free(fd->io_buf); - /* If one-sided aggregation is chosen then free the window over the io_buf. - */ - if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { - MPI_Win_free(&fd->io_buf_window); - if (!gpfsmpio_onesided_no_rmw) { - MPI_Win_free(&fd->io_buf_put_amounts_window); - ADIOI_Free(fd->io_buf_put_amounts); - } - } + ADIOI_OneSidedCleanup(fd); /* memory for fd is freed in MPI_File_close */ } diff --git a/src/mpi/romio/adio/common/ad_open.c b/src/mpi/romio/adio/common/ad_open.c index 222cb55..5c95b47 100644 --- a/src/mpi/romio/adio/common/ad_open.c +++ b/src/mpi/romio/adio/common/ad_open.c @@ -10,8 +10,6 @@ #include "adio_cb_config_list.h" #include "mpio.h" -extern int gpfsmpio_aggmethod; -extern int gpfsmpio_onesided_no_rmw; static int is_aggregator(int rank, ADIO_File fd); static int uses_generic_read(ADIO_File fd); static int uses_generic_write(ADIO_File fd); @@ -72,6 +70,9 @@ MPI_File ADIO_Open(MPI_Comm orig_comm, fd->err_handler = ADIOI_DFLT_ERR_HANDLER; + fd->io_buf_window = MPI_WIN_NULL; + fd->io_buf_put_amounts_window = MPI_WIN_NULL; + MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &procs); /* create and initialize info object */ @@ -127,15 +128,6 @@ MPI_File ADIO_Open(MPI_Comm orig_comm, * (e.g. Blue Gene) more efficent */ fd->io_buf = ADIOI_Malloc(fd->hints->cb_buffer_size); - /* If one-sided aggregation is chosen then create the window over the io_buf. - */ - if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { - MPI_Win_create(fd->io_buf,fd->hints->cb_buffer_size,1,MPI_INFO_NULL,fd->comm, &fd->io_buf_window); - if (!gpfsmpio_onesided_no_rmw) { - fd->io_buf_put_amounts = (int *) ADIOI_Malloc(procs*sizeof(int)); - MPI_Win_create(fd->io_buf_put_amounts,procs*sizeof(int),sizeof(int),MPI_INFO_NULL,fd->comm, &fd->io_buf_put_amounts_window); - } - } /* deferred open: * we can only do this optimization if 'fd->hints->deferred_open' is set * (which means the user hinted 'no_indep_rw' and collective buffering). diff --git a/src/mpi/romio/adio/common/onesided_aggregation.c b/src/mpi/romio/adio/common/onesided_aggregation.c index 559ea40..84d8333 100644 --- a/src/mpi/romio/adio/common/onesided_aggregation.c +++ b/src/mpi/romio/adio/common/onesided_aggregation.c @@ -1,6 +1,12 @@ #include "adio.h" #include "adio_extern.h" +#ifdef ROMIO_GPFS +/* right now this is GPFS only but TODO: extend this to all file systems */ #include "../ad_gpfs/ad_gpfs_tuning.h" +#else +int gpfsmpio_onesided_no_rmw = 0; +int gpfsmpio_aggmethod = 0; +#endif #include <pthread.h> @@ -20,6 +26,32 @@ typedef struct NonContigSourceBufOffset { ADIO_Offset indiceOffset; } NonContigSourceBufOffset; +static int ADIOI_OneSidedSetup(ADIO_File fd, int procs) { + int ret = MPI_SUCCESS; + + ret = MPI_Win_create(fd->io_buf,fd->hints->cb_buffer_size,1, + MPI_INFO_NULL,fd->comm, &fd->io_buf_window); + if (ret != MPI_SUCCESS) goto fn_exit; + fd->io_buf_put_amounts = (int *) ADIOI_Malloc(procs*sizeof(int)); + ret =MPI_Win_create(fd->io_buf_put_amounts,procs*sizeof(int),sizeof(int), + MPI_INFO_NULL,fd->comm, &fd->io_buf_put_amounts_window); +fn_exit: + return ret; +} + +int ADIOI_OneSidedCleanup(ADIO_File fd) +{ + int ret = MPI_SUCCESS; + if (fd->io_buf_window != MPI_WIN_NULL) + ret = MPI_Win_free(&fd->io_buf_window); + if (fd->io_buf_put_amounts_window != MPI_WIN_NULL) + ret = MPI_Win_free(&fd->io_buf_put_amounts_window); + if (fd->io_buf_put_amounts != NULL) + ADIOI_Free(fd->io_buf_put_amounts); + + return ret; +} + void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset *offset_list, ADIO_Offset *len_list, @@ -51,6 +83,12 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, MPI_Comm_size(fd->comm, &nprocs); MPI_Comm_rank(fd->comm, &myrank); + if (fd->io_buf_window == MPI_WIN_NULL || + fd->io_buf_put_amounts_window == MPI_WIN_NULL) + { + ADIOI_OneSidedSetup(fd, nprocs); + } + /* This flag denotes whether the source datatype is contiguous, which is referenced throughout the algorithm * and defines how the source buffer offsets and data chunks are determined. If the value is 1 (true - contiguous data) * things are profoundly simpler in that the source buffer offset for a given target offset simply linearly increases @@ -1063,6 +1101,11 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, MPI_Comm_size(fd->comm, &nprocs); MPI_Comm_rank(fd->comm, &myrank); + if (fd->io_buf_window == MPI_WIN_NULL || + fd->io_buf_put_amounts_window == MPI_WIN_NULL) + { + ADIOI_OneSidedSetup(fd, nprocs); + } /* This flag denotes whether the source datatype is contiguus, which is referenced throughout the algorithm * and defines how the source buffer offsets and data chunks are determined. If the value is 1 (true - contiguous data) * things are profoundly simpler in that the source buffer offset for a given source offset simply linearly increases diff --git a/src/mpi/romio/adio/include/adioi.h b/src/mpi/romio/adio/include/adioi.h index 0290cb4..d2dbc94 100644 --- a/src/mpi/romio/adio/include/adioi.h +++ b/src/mpi/romio/adio/include/adioi.h @@ -686,6 +686,7 @@ void ADIOI_P2PContigReadAggregation(ADIO_File fd, ADIO_Offset *fd_start, ADIO_Offset *fd_end); +int ADIOI_OneSidedCleanup(ADIO_File fd); void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset *offset_list, ADIO_Offset *len_list, http://git.mpich.org/mpich.git/commitdiff/5c4f21d038c2975d968af232447b25aaf… commit 5c4f21d038c2975d968af232447b25aaf6d1442d Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Sat Feb 28 01:33:02 2015 +0000 PAMI_Rput_typed / PAMI_Rget_typed utilization for derived types Optimization to use the PAMI_Rput_typed / PAMI_Rget_typed call in the case where PAMID MPI_Put / MPI_Get is called with a derived (non-contiguous) datatype. Instead of breaking the MPI datatype up into contiguous chunks on the MPICH side and repeatedly calling PAMI_Rput / PAMI_Rget for each chunk with the associated overhead, create a PAMI datatype to represent the MPI derived type and make just 1 call to PAMI_Rput_typed / PAMI_Rget_typed. We deal with non-contiguous buffers by avoiding packing and using origin buffers (as in PAMI) Guarded by the PAMID_TYPED_ONESIDED environment variable. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpid/common/datatype/mpid_type_commit.c b/src/mpid/common/datatype/mpid_type_commit.c index d990503..28be988 100644 --- a/src/mpid/common/datatype/mpid_type_commit.c +++ b/src/mpid/common/datatype/mpid_type_commit.c @@ -30,7 +30,7 @@ int MPID_Type_commit(MPI_Datatype *datatype_p) MPID_Datatype_get_ptr(*datatype_p, datatype_ptr); if (datatype_ptr->is_committed == 0) { - datatype_ptr->is_committed = 1; + datatype_ptr->is_committed = 1; #ifdef MPID_NEEDS_DLOOP_ALL_BYTES /* If MPID implementation needs use to reduce everything to @@ -62,8 +62,11 @@ int MPID_Type_commit(MPI_Datatype *datatype_p) MPIDI_Dataloop_dot_printf(datatype_ptr->dataloop, 0, 1); #endif - } +#ifdef MPID_Dev_datatype_commit_hook + MPID_Dev_datatype_commit_hook(datatype_p); +#endif /* MPID_Dev_datatype_commit_hook */ + } return mpi_errno; } diff --git a/src/mpid/common/datatype/mpid_type_dup.c b/src/mpid/common/datatype/mpid_type_dup.c index ae24dad..fddabea 100644 --- a/src/mpid/common/datatype/mpid_type_dup.c +++ b/src/mpid/common/datatype/mpid_type_dup.c @@ -82,7 +82,6 @@ int MPID_Type_dup(MPI_Datatype oldtype, new_dtp->hetero_dloop = NULL; new_dtp->hetero_dloop_size = old_dtp->hetero_dloop_size; new_dtp->hetero_dloop_depth = old_dtp->hetero_dloop_depth; - *newtype = new_dtp->handle; if (old_dtp->is_committed) { @@ -98,7 +97,11 @@ int MPID_Type_dup(MPI_Datatype oldtype, old_dtp->hetero_dloop_size, &new_dtp->hetero_dloop); } - } + +#ifdef MPID_Dev_datatype_commit_hook + MPID_Dev_datatype_dup_hook(new_dtp); +#endif /* MPID_Dev_datatype_commit_hook */ + } } MPIU_DBG_MSG_D(DATATYPE,VERBOSE, "dup type %x created.", *newtype); diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 14e8dc8..d3e4797 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -148,6 +148,7 @@ typedef struct unsigned mpir_nbc; /**< Enable MPIR_* non-blocking collectives implementations. */ int numTasks; /* total number of tasks on a job */ + unsigned typed_onesided; /**< Enable typed PAMI calls for derived types within MPID_Put and MPID_Get. */ #ifdef DYNAMIC_TASKING struct MPIDI_PG_t * my_pg; /**< Process group I belong to */ int my_pg_rank; /**< Rank in process group */ diff --git a/src/mpid/pamid/include/mpidpre.h b/src/mpid/pamid/include/mpidpre.h index 29039e2..a10e1ea 100644 --- a/src/mpid/pamid/include/mpidpre.h +++ b/src/mpid/pamid/include/mpidpre.h @@ -70,5 +70,13 @@ #define MPID_MAX_SMP_BCAST_MSG_SIZE (16384) #define MPID_MAX_SMP_REDUCE_MSG_SIZE (16384) #define MPID_MAX_SMP_ALLREDUCE_MSG_SIZE (16384) +#ifdef MPID_DEV_DATATYPE_DECL +#error 'Conflicting definitions of MPID_DEV_DATATYPE_DECL' +#else +#define MPID_DEV_DATATYPE_DECL void *device_datatype; +#endif +#define MPID_Dev_datatype_commit_hook(ptr) MPIDI_PAMI_datatype_commit_hook(ptr) +#define MPID_Dev_datatype_destroy_hook(ptr) MPIDI_PAMI_datatype_destroy_hook(ptr) +#define MPID_Dev_datatype_dup_hook(ptr) MPIDI_PAMI_datatype_dup_hook(ptr) #endif diff --git a/src/mpid/pamid/src/Makefile.mk b/src/mpid/pamid/src/Makefile.mk index a8d98e1..e2468f3 100644 --- a/src/mpid/pamid/src/Makefile.mk +++ b/src/mpid/pamid/src/Makefile.mk @@ -59,7 +59,8 @@ mpi_core_sources += \ src/mpid/pamid/src/mpid_imrecv.c \ src/mpid/pamid/src/mpid_improbe.c \ src/mpid/pamid/src/mpid_aint.c \ - src/mpid/pamid/src/mpidi_nbc_sched.c + src/mpid/pamid/src/mpidi_nbc_sched.c \ + src/mpid/pamid/src/mpidi_pami_datatype.c if QUEUE_BINARY_SEARCH_SUPPORT mpi_core_sources += \ diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index e768119..a2bc89d 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -135,6 +135,7 @@ MPIDI_Process_t MPIDI_Process = { .mpir_nbc = 1, .numTasks = 0, + .typed_onesided = 0, }; @@ -1016,6 +1017,7 @@ MPIDI_PAMI_init(int* rank, int* size, int* threading) " optimized.num_requests: %u\n" " mpir_nbc : %u\n" " numTasks : %u\n", + " typed_onesided : %u\n", MPIDI_Process.verbose, MPIDI_Process.statistics, MPIDI_Process.avail_contexts, @@ -1052,7 +1054,8 @@ MPIDI_PAMI_init(int* rank, int* size, int* threading) MPIDI_Process.optimized.memory, MPIDI_Process.optimized.num_requests, MPIDI_Process.mpir_nbc, - MPIDI_Process.numTasks); + MPIDI_Process.numTasks, + MPIDI_Process.typed_onesided); switch (*threading) { case MPI_THREAD_MULTIPLE: diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c index 3eb1aac..f6934c6 100644 --- a/src/mpid/pamid/src/mpidi_env.c +++ b/src/mpid/pamid/src/mpidi_env.c @@ -938,6 +938,11 @@ MPIDI_Env_setup(int rank, int requested) ENV_Unsigned(names, &MPIDI_Process.mpir_nbc, 1, &found_deprecated_env_var, rank); } + /* Enable typed PAMI calls for derived types within MPID_Put and MPID_Get. */ + { + char* names[] = {"PAMID_TYPED_ONESIDED", NULL}; + ENV_Unsigned(names, &MPIDI_Process.typed_onesided, 1, &found_deprecated_env_var, rank); + } /* Check for deprecated collectives environment variables. These variables are * used in src/mpid/pamid/src/comm/mpid_selectcolls.c */ { diff --git a/src/mpid/pamid/src/mpidi_pami_datatype.c b/src/mpid/pamid/src/mpidi_pami_datatype.c new file mode 100644 index 0000000..b700ea0 --- /dev/null +++ b/src/mpid/pamid/src/mpidi_pami_datatype.c @@ -0,0 +1,155 @@ +/* begin_generated_IBM_copyright_prolog */ +/* */ +/* This is an automatically generated copyright prolog. */ +/* After initializing, DO NOT MODIFY OR MOVE */ +/* --------------------------------------------------------------- */ +/* Licensed Materials - Property of IBM */ +/* Blue Gene/Q 5765-PER 5765-PRP */ +/* */ +/* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */ +/* US Government Users Restricted Rights - */ +/* Use, duplication, or disclosure restricted */ +/* by GSA ADP Schedule Contract with IBM Corp. */ +/* */ +/* --------------------------------------------------------------- */ +/* */ +/* end_generated_IBM_copyright_prolog */ +/* (C)Copyright IBM Corp. 2007, 2011 */ +/** + * \file src/mpidi_pami_datatype.c + * \brief pami_type_t datatype hooks + */ + +#include <pami.h> +#include <mpidimpl.h> + +/** + * \brief Create PAMI datatype representation of MPI Datatype during commit. + * + * Signifcant performance improvements can be realized for one-sided communication + * utilizing the PAMI_Rput_typed and PAMI_Rget_typed interface which requires a + * PAMI representation of the MPI Datatype. + */ +void MPIDI_PAMI_datatype_commit_hook (MPI_Datatype *ptr) +{ + + /* If the PAMID optimization to utilize the PAMI_Rput_typed / PAMI_Rget_typed call for + * one-sided comm for derived types is enabled then we need to create the PAMI datatype. + */ + if (MPIDI_Process.typed_onesided == 1) { + + MPID_Datatype *datatype_ptr; + MPID_Datatype_get_ptr(*ptr, datatype_ptr); + + pami_result_t pami_dtop_result; + datatype_ptr->device_datatype = (pami_type_t *) MPIU_Malloc(sizeof(pami_type_t)); + pami_dtop_result = PAMI_Type_create ((pami_type_t *)datatype_ptr->device_datatype); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + + /* Flatten the non-contiguous data type into arrays describing the contiguous chunks. + */ + MPI_Aint *dt_offset_array = (MPI_Aint *) MPIU_Malloc(datatype_ptr->max_contig_blocks * sizeof(MPI_Aint)); + MPI_Aint *dt_size_array = (MPI_Aint *) MPIU_Malloc(datatype_ptr->max_contig_blocks * sizeof(MPI_Aint)); + MPI_Aint dt_array_len = datatype_ptr->max_contig_blocks; + int rc = MPIR_Type_flatten(*ptr, dt_offset_array, dt_size_array, &dt_array_len); + + /* Build the PAMI datatype adding one contiguous chunk at a time with the PAMI_Type_add_simple + * interface. + */ + int i; + + for (i=0;i<dt_array_len;i++) { + size_t num_bytes_this_entry = dt_size_array[i]; + size_t cursor_offset; + if (i == 0) + cursor_offset = (size_t) dt_offset_array[i]; + else + cursor_offset = (size_t) dt_offset_array[i] - (size_t)dt_offset_array[i-1]; + pami_dtop_result = PAMI_Type_add_simple (*(pami_type_t*)(datatype_ptr->device_datatype), num_bytes_this_entry, cursor_offset, 1, 0); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + } + + /* Complete the PAMI datatype and free arrays. + */ + pami_dtop_result = PAMI_Type_complete (*(pami_type_t*)(datatype_ptr->device_datatype),1); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + MPIU_Free(dt_offset_array); + MPIU_Free(dt_size_array); + } + return; +} + +/** + * \brief Destroy PAMI datatype representation of MPI Datatype. + * + */ +void MPIDI_PAMI_datatype_destroy_hook (MPID_Datatype *ptr) +{ + /* If a PAMI datatype was created, destroy it if this is the + * last reference to the MPID_Datatype ptr. + */ + if ((MPIDI_Process.typed_onesided == 1) && (ptr->is_committed)) { + if (ptr->device_datatype) { + pami_result_t pami_dtop_result; + pami_dtop_result = PAMI_Type_destroy ((pami_type_t *)ptr->device_datatype); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + MPIU_Free(ptr->device_datatype); + } + } +} + +/** + * \brief Create PAMI datatype representation of MPI Datatype during dup. + * + * Signifcant performance improvements can be realized for one-sided communication + * utilizing the PAMI_Rput_typed and PAMI_Rget_typed interface which requires a + * PAMI representation of the MPI Datatype. + */ +void MPIDI_PAMI_datatype_dup_hook (MPI_Datatype *ptr) +{ + + /* If the PAMID optimization to utilize the PAMI_Rput_typed / PAMI_Rget_typed call for + * one-sided comm for derived types is enabled then we need to create the PAMI datatype. + */ + if (MPIDI_Process.typed_onesided == 1) { + + MPID_Datatype *datatype_ptr; + MPID_Datatype_get_ptr(*ptr, datatype_ptr); + + pami_result_t pami_dtop_result; + datatype_ptr->device_datatype = (pami_type_t *) MPIU_Malloc(sizeof(pami_type_t)); + pami_dtop_result = PAMI_Type_create ((pami_type_t *)datatype_ptr->device_datatype); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + + /* Flatten the non-contiguous data type into arrays describing the contiguous chunks. + */ + MPI_Aint *dt_offset_array = (MPI_Aint *) MPIU_Malloc(datatype_ptr->max_contig_blocks * sizeof(MPI_Aint)); + MPI_Aint *dt_size_array = (MPI_Aint *) MPIU_Malloc(datatype_ptr->max_contig_blocks * sizeof(MPI_Aint)); + MPI_Aint dt_array_len = datatype_ptr->max_contig_blocks; + int rc = MPIR_Type_flatten(*ptr, dt_offset_array, dt_size_array, &dt_array_len); + + /* Build the PAMI datatype adding one contiguous chunk at a time with the PAMI_Type_add_simple + * interface. + */ + int i; + + for (i=0;i<dt_array_len;i++) { + size_t num_bytes_this_entry = dt_size_array[i]; + size_t cursor_offset; + if (i == 0) + cursor_offset = (size_t) dt_offset_array[i]; + else + cursor_offset = (size_t) dt_offset_array[i] - (size_t)dt_offset_array[i-1]; + pami_dtop_result = PAMI_Type_add_simple (*(pami_type_t*)(datatype_ptr->device_datatype), num_bytes_this_entry, cursor_offset, 1, 0); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + } + + /* Complete the PAMI datatype and free arrays. + */ + pami_dtop_result = PAMI_Type_complete (*(pami_type_t*)(datatype_ptr->device_datatype),1); + MPIU_Assert(pami_dtop_result == PAMI_SUCCESS); + MPIU_Free(dt_offset_array); + MPIU_Free(dt_size_array); + } + return; +} diff --git a/src/mpid/pamid/src/onesided/mpid_win_get.c b/src/mpid/pamid/src/onesided/mpid_win_get.c index 872f62f..ba318af 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get.c @@ -22,7 +22,6 @@ #include "mpidi_onesided.h" #include "mpidi_util.h" - static inline int MPIDI_Get_use_pami_rget(pami_context_t context, MPIDI_Win_request * req) __attribute__((__always_inline__)); @@ -60,6 +59,39 @@ MPIDI_Get(pami_context_t context, static inline int MPIDI_Get_use_pami_rget(pami_context_t context, MPIDI_Win_request * req) { + int use_typed_rdma = 0; + + if (!req->target.dt.contig || !req->origin.dt.contig) { + use_typed_rdma = 0; + if (MPIDI_Process.typed_onesided == 1) + use_typed_rdma = 1; + } + + if (use_typed_rdma) { + pami_result_t rc; + pami_rget_typed_t params; + /* params need to zero out to avoid passing garbage to PAMI */ + params=zero_rget_typed_parms; + + params.rma.dest=req->dest; + params.rma.hints.buffer_registered = PAMI_HINT_ENABLE; + params.rma.hints.use_rdma = PAMI_HINT_ENABLE; + params.rma.bytes = req->target.dt.size; + params.rma.cookie = req; + params.rma.done_fn = MPIDI_Win_DoneCB; + params.rdma.local.mr=&req->origin.memregion; + params.rdma.remote.mr=&req->win->mpid.info[req->target.rank].memregion; + params.rdma.remote.offset= req->offset; + params.rdma.local.offset = req->state.local_offset; + + params.type.local = *(pami_type_t *)(req->origin.dt.pointer->device_datatype); + params.type.remote = *(pami_type_t *)(req->target.dt.pointer->device_datatype); + + + rc = PAMI_Rget_typed(context, &params); + MPID_assert(rc == PAMI_SUCCESS); + } + else { pami_result_t rc; pami_rget_simple_t params; @@ -110,6 +142,7 @@ MPIDI_Get_use_pami_rget(pami_context_t context, MPIDI_Win_request * req) ++req->state.index; } } + } return PAMI_SUCCESS; } @@ -203,6 +236,7 @@ MPID_Get(void *origin_addr, MPI_Datatype target_datatype, MPID_Win *win) { + int mpi_errno = MPI_SUCCESS; int shm_locked=0; void *target_addr; @@ -293,10 +327,17 @@ MPID_Get(void *origin_addr, req->target.rank = target_rank; - if (req->origin.dt.contig) + /* Only pack the origin data if the origin is non-contiguous and we are using the simple PAMI_Rget. + * If we are using the typed PAMI_Rget_typed use the origin address as is, if we are using the simple + * PAMI_Rget with contiguous data use the origin address with the lower-bound adjustment. + */ + if (req->origin.dt.contig || (!req->origin.dt.contig && (MPIDI_Process.typed_onesided == 1))) { req->buffer_free = 0; - req->buffer = origin_addr + req->origin.dt.true_lb; + if ((req->origin.dt.contig && req->target.dt.contig && (MPIDI_Process.typed_onesided == 1)) || (!(MPIDI_Process.typed_onesided == 1))) // use simple rput + req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); + else + req->buffer = (void *) ((uintptr_t) origin_addr); } else { @@ -356,8 +397,14 @@ MPID_Get(void *origin_addr, MPIDI_Win_datatype_map(&req->target.dt); - win->mpid.sync.total += req->target.dt.num_contig; + if ((!req->target.dt.contig || !req->origin.dt.contig) && (MPIDI_Process.typed_onesided == 1)) + /* If the datatype is non-contiguous and the PAMID typed_onesided optimization + * is enabled then we will be using the typed interface and will only make 1 call. + */ + win->mpid.sync.total = 1; + else + win->mpid.sync.total += req->target.dt.num_contig; /* The pamid one-sided design requires context post in order to handle the * case where the number of pending rma operation exceeds the diff --git a/src/mpid/pamid/src/onesided/mpid_win_put.c b/src/mpid/pamid/src/onesided/mpid_win_put.c index 9cbcc05..a7e7ce5 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_put.c +++ b/src/mpid/pamid/src/onesided/mpid_win_put.c @@ -61,6 +61,39 @@ MPIDI_Put(pami_context_t context, static inline int MPIDI_Put_use_pami_rput(pami_context_t context, MPIDI_Win_request * req) { + int use_typed_rdma = 0; + if (!req->target.dt.contig || !req->origin.dt.contig) { + use_typed_rdma = 0; + if (MPIDI_Process.typed_onesided == 1) + use_typed_rdma = 1; + } + + if (use_typed_rdma) { + pami_result_t rc; + pami_rput_typed_t params; + /* params need to zero out to avoid passing garbage to PAMI */ + params=zero_rput_typed_parms; + + params.rma.dest=req->dest; + params.rma.hints.buffer_registered = PAMI_HINT_ENABLE; + params.rma.hints.use_rdma = PAMI_HINT_ENABLE; + params.rma.bytes = req->target.dt.size; + params.rma.cookie = req; + params.rma.done_fn = NULL; + params.rdma.local.mr=&req->origin.memregion; + params.rdma.remote.mr=&req->win->mpid.info[req->target.rank].memregion; + params.rdma.remote.offset= req->offset; + params.rdma.local.offset = req->state.local_offset; + params.put.rdone_fn= MPIDI_Win_DoneCB; + + params.type.local = *(pami_type_t *)(req->origin.dt.pointer->device_datatype); + params.type.remote = *(pami_type_t *)(req->target.dt.pointer->device_datatype); + + rc = PAMI_Rput_typed(context, &params); + MPID_assert(rc == PAMI_SUCCESS); + + } + else { pami_result_t rc; pami_rput_simple_t params; /* params need to zero out to avoid passing garbage to PAMI */ @@ -113,6 +146,7 @@ MPIDI_Put_use_pami_rput(pami_context_t context, MPIDI_Win_request * req) ++req->state.index; } } + } return PAMI_SUCCESS; } @@ -296,10 +330,17 @@ MPID_Put(const void *origin_addr, req->target.rank = target_rank; - if (req->origin.dt.contig) + /* Only pack the origin data if the origin is non-contiguous and we are using the simple PAMI_Rput. + * If we are using the typed PAMI_Rput_typed use the origin address as-is, if we are using the simple + * PAMI_Rput with contiguous data use the origin address with the lower-bound adjustment. + */ + if (req->origin.dt.contig || (!req->origin.dt.contig && (MPIDI_Process.typed_onesided == 1))) { req->buffer_free = 0; - req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); + if ((req->origin.dt.contig && req->target.dt.contig && (MPIDI_Process.typed_onesided == 1)) || (!(MPIDI_Process.typed_onesided == 1))) + req->buffer = (void *) ((uintptr_t) origin_addr + req->origin.dt.true_lb); + else + req->buffer = (void *) ((uintptr_t) origin_addr); } else { @@ -357,7 +398,13 @@ MPID_Put(const void *origin_addr, MPIDI_Win_datatype_map(&req->target.dt); - win->mpid.sync.total += req->target.dt.num_contig; + if ((!req->target.dt.contig || !req->origin.dt.contig) && (MPIDI_Process.typed_onesided == 1)) + /* If the datatype is non-contiguous and the PAMID typed_onesided optimization + * is enabled then we will be using the typed interface and will only make 1 call. + */ + win->mpid.sync.total = 1; + else + win->mpid.sync.total += req->target.dt.num_contig; /* The pamid one-sided design requires context post in order to handle the * case where the number of pending rma operation exceeds the diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 0b324cc..23e2cf8 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -29,6 +29,8 @@ pami_rget_simple_t zero_rget_parms; pami_get_simple_t zero_get_parms; pami_rput_simple_t zero_rput_parms; pami_put_simple_t zero_put_parms; +pami_rput_typed_t zero_rput_typed_parms; +pami_rget_typed_t zero_rget_typed_parms; pami_send_t zero_send_parms; pami_send_immediate_t zero_send_immediate_parms; pami_recv_t zero_recv_parms; http://git.mpich.org/mpich.git/commitdiff/18516bafa3a3ee7a9e0343ad1bda9876f… commit 18516bafa3a3ee7a9e0343ad1bda9876fa52a49a Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Mon Mar 23 18:39:49 2015 -0500 ROMIO Collective IO One-sided aggregation algorithm support read-modify-write Added support to additionally run two-phase aggregation which has the read-modify-write capability in cases where the one-sided write aggregation encounters holes in the data. Additon of two new environment variables (GPFSMPIO_ONESIDED_NO_RMW, GPFSMPIO_ONESIDED_INFORM_RMW) to control this behavior and inform the user. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c index 192f747..0d44176 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c @@ -41,6 +41,8 @@ int gpfsmpio_aggmethod; int gpfsmpio_balancecontig; int gpfsmpio_devnullio; int gpfsmpio_bridgeringagg; +int gpfsmpio_onesided_no_rmw; +int gpfsmpio_onesided_inform_rmw; double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST+1]; double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; @@ -122,6 +124,22 @@ double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; * optimal performance for this is achieved when paired with PAMID_TYPED_ONESIDED=1. * - Default is 0 * + * - GPFSMPIO_ONESIDED_NO_RMW - For one-sided aggregation (GPFSMPIO_AGGMETHOD = 1 or 2) + * disable the detection of holes in the data when writing to a pre-existing + * file requiring a read-modify-write, thereby avoiding the communication + * overhead for this detection. + * - 0 (hole detection enabled) or 1 (hole detection disabled) + * - Default is 0 + * + * - GPFSMPIO_ONESIDED_INFORM_RMW - For one-sided aggregation + * (GPFSMPIO_AGGMETHOD = 1 or 2) generate an informational message informing + * the user whether holes exist in the data when writing to a pre-existing + * file requiring a read-modify-write, thereby educating the user to set + * GPFSMPIO_ONESIDED_NO_RMW=1 on a future run to avoid the communication + * overhead for this detection. + * - 0 (disabled) or 1 (enabled) + * - Default is 0 + * * - GPFSMPIO_BALANCECONTIG - Relevant only to BGQ. File domain blocks are assigned * to aggregators in a breadth-first fashion relative to the ions - additionally, * file domains on the aggregators sharing the same bridgeset and ion have contiguous @@ -197,6 +215,14 @@ void ad_gpfs_get_env_vars() { gpfsmpio_bridgeringagg = 0; x = getenv( "GPFSMPIO_BRIDGERINGAGG" ); if (x) gpfsmpio_bridgeringagg = atoi(x); + + gpfsmpio_onesided_no_rmw = 0; + x = getenv( "GPFSMPIO_ONESIDED_NO_RMW" ); + if (x) gpfsmpio_onesided_no_rmw = atoi(x); + + gpfsmpio_onesided_inform_rmw = 0; + x = getenv( "GPFSMPIO_ONESIDED_INFORM_RMW" ); + if (x) gpfsmpio_onesided_inform_rmw = atoi(x); } /* report timing breakdown for MPI I/O collective call */ diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h index 87aefce..56e1588 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h @@ -23,6 +23,7 @@ * Global variables for the control of * 1. timing * 2. select specific optimizations + * 3. global flags for certain optimizations *-----------------------------------------*/ /* timing fields */ @@ -56,7 +57,6 @@ enum { extern double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST+1]; extern double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; - /* corresponds to environment variables to select optimizations and timing level */ extern int gpfsmpio_timing; extern int gpfsmpio_timing_cw_level; @@ -70,6 +70,8 @@ extern int gpfsmpio_aggmethod; extern int gpfsmpio_balancecontig; extern int gpfsmpio_devnullio; extern int gpfsmpio_bridgeringagg; +extern int gpfsmpio_onesided_no_rmw; +extern int gpfsmpio_onesided_inform_rmw; /* Default is, well, kind of complicated. Blue Gene /L and /P had "psets": one * i/o node and all compute nodes wired to it. On Blue Gene /Q that diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 9ce68fa..4fcb14c 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -267,7 +267,12 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count, /* If the user has specified to use a one-sided aggregation method then do that at * this point instead of the two-phase I/O. */ - ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count, buf, datatype, error_code, st_offsets, end_offsets, fd_start, fd_end); + int holeFound = 0; + ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count, buf, datatype, error_code, st_offsets, end_offsets, fd_start, fd_end, &holeFound); + int anyHolesFound = 0; + if (!gpfsmpio_onesided_no_rmw) + MPI_Allreduce(&holeFound, &anyHolesFound, 1, MPI_INT, MPI_MAX, fd->comm); + if (anyHolesFound == 0) { GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs) ADIOI_Free(offset_list); ADIOI_Free(len_list); @@ -276,6 +281,15 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count, ADIOI_Free(fd_start); ADIOI_Free(fd_end); goto fn_exit; + } + else { + /* Holes are found in the data and the user has not set gpfsmpio_onesided_no_rmw --- + * fall thru and perform the two-phase aggregation and if the user has gpfsmpio_onesided_inform_rmw + * set then inform him of this condition and behavior. + */ + if (gpfsmpio_onesided_inform_rmw && (myrank ==0)) + FPRINTF(stderr,"Information: Holes found during one-sided write aggregation algorithm --- additionally performing default two-phase aggregation algorithm\n"); + } } if (gpfsmpio_p2pcontig==1) { /* For some simple yet common(?) workloads, full-on two-phase I/O is overkill. We can establish sub-groups of processes and their aggregator, and then these sub-groups will carry out a simplified two-phase over that sub-group. diff --git a/src/mpi/romio/adio/common/ad_close.c b/src/mpi/romio/adio/common/ad_close.c index d51d75c..ff0049e 100644 --- a/src/mpi/romio/adio/common/ad_close.c +++ b/src/mpi/romio/adio/common/ad_close.c @@ -8,6 +8,7 @@ #include "adio.h" #include "adio_extern.h" extern int gpfsmpio_aggmethod; +extern int gpfsmpio_onesided_no_rmw; #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -118,8 +119,13 @@ void ADIO_Close(ADIO_File fd, int *error_code) if (fd->io_buf != NULL) ADIOI_Free(fd->io_buf); /* If one-sided aggregation is chosen then free the window over the io_buf. */ - if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) + if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { MPI_Win_free(&fd->io_buf_window); + if (!gpfsmpio_onesided_no_rmw) { + MPI_Win_free(&fd->io_buf_put_amounts_window); + ADIOI_Free(fd->io_buf_put_amounts); + } + } /* memory for fd is freed in MPI_File_close */ } diff --git a/src/mpi/romio/adio/common/ad_open.c b/src/mpi/romio/adio/common/ad_open.c index c0d9331..222cb55 100644 --- a/src/mpi/romio/adio/common/ad_open.c +++ b/src/mpi/romio/adio/common/ad_open.c @@ -11,6 +11,7 @@ #include "mpio.h" extern int gpfsmpio_aggmethod; +extern int gpfsmpio_onesided_no_rmw; static int is_aggregator(int rank, ADIO_File fd); static int uses_generic_read(ADIO_File fd); static int uses_generic_write(ADIO_File fd); @@ -124,12 +125,16 @@ MPI_File ADIO_Open(MPI_Comm orig_comm, /* Instead of repeatedly allocating this buffer in collective read/write, * allocating up-front might make memory management on small platforms * (e.g. Blue Gene) more efficent */ - fd->io_buf = ADIOI_Malloc(fd->hints->cb_buffer_size); + fd->io_buf = ADIOI_Malloc(fd->hints->cb_buffer_size); /* If one-sided aggregation is chosen then create the window over the io_buf. */ if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { MPI_Win_create(fd->io_buf,fd->hints->cb_buffer_size,1,MPI_INFO_NULL,fd->comm, &fd->io_buf_window); + if (!gpfsmpio_onesided_no_rmw) { + fd->io_buf_put_amounts = (int *) ADIOI_Malloc(procs*sizeof(int)); + MPI_Win_create(fd->io_buf_put_amounts,procs*sizeof(int),sizeof(int),MPI_INFO_NULL,fd->comm, &fd->io_buf_put_amounts_window); + } } /* deferred open: * we can only do this optimization if 'fd->hints->deferred_open' is set diff --git a/src/mpi/romio/adio/common/onesided_aggregation.c b/src/mpi/romio/adio/common/onesided_aggregation.c index 50d9315..559ea40 100644 --- a/src/mpi/romio/adio/common/onesided_aggregation.c +++ b/src/mpi/romio/adio/common/onesided_aggregation.c @@ -30,9 +30,10 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset *st_offsets, ADIO_Offset *end_offsets, ADIO_Offset *fd_start, - ADIO_Offset* fd_end) -{ + ADIO_Offset* fd_end, + int *hole_found) +{ *error_code = MPI_SUCCESS; /* initialize to success */ #ifdef ROMIO_GPFS @@ -119,10 +120,6 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset greatestFileDomainOffset = 0; ADIO_Offset smallestFileDomainOffset = lastFileOffset; for (j=0;j<naggs;j++) { - /* Find the actual lowest and highest offsets to be written. - The non-aggs need to know this too to adjust the mpi_put - window displacement accordingly. - */ if (fd_end[j] > greatestFileDomainOffset) { greatestFileDomainOffset = fd_end[j]; greatestFileDomainAggRank = j; @@ -346,6 +343,12 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, targetAggsForMyDataFDStart[numTargetAggs] = firstFileOffset; } targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex]; + /* Round down file domain to the last actual offset used if this is the last file domain. + */ + if (currentAggRankListIndex == greatestFileDomainAggRank) { + if (targetAggsForMyDataFDEnd[numTargetAggs] > lastFileOffset) + targetAggsForMyDataFDEnd[numTargetAggs] = lastFileOffset; + } targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; if (bufTypeIsContig) baseSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = currentSourceBufferOffset; @@ -417,6 +420,12 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, targetAggsForMyDataFDStart[numTargetAggs] = firstFileOffset; } targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex]; + /* Round down file domain to the last actual offset used if this is the last file domain. + */ + if (currentAggRankListIndex == greatestFileDomainAggRank) { + if (targetAggsForMyDataFDEnd[numTargetAggs] > lastFileOffset) + targetAggsForMyDataFDEnd[numTargetAggs] = lastFileOffset; + } targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; if (bufTypeIsContig) baseSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = currentSourceBufferOffset; @@ -493,8 +502,16 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, char *write_buf = write_buf0; MPI_Win write_buf_window = fd->io_buf_window; + int *write_buf_put_amounts = fd->io_buf_put_amounts; + if(!gpfsmpio_onesided_no_rmw) { + *hole_found = 0; + for (i=0;i<nprocs;i++) + write_buf_put_amounts[i] = 0; + } #ifdef ACTIVE_TARGET MPI_Win_fence(0, write_buf_window); + if (!gpfsmpio_onesided_no_rmw) + MPI_Win_fence(0, fd->io_buf_put_amounts_window); #endif ADIO_Offset currentRoundFDStart = 0; @@ -506,6 +523,10 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, if (currentRoundFDStart < firstFileOffset) currentRoundFDStart = firstFileOffset; } + else if (myAggRank == greatestFileDomainAggRank) { + if (currentRoundFDEnd > lastFileOffset) + currentRoundFDEnd = lastFileOffset; + } } #ifdef ROMIO_GPFS @@ -537,6 +558,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, int aggIter; for (aggIter=0;aggIter<numTargetAggs;aggIter++) { + int numBytesPutThisAggRound = 0; /* If we have data for the round/agg process it. */ if ((bufTypeIsContig && (baseSourceBufferOffset[roundIter][aggIter] != -1)) || (!bufTypeIsContig && (baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice != -1))) { @@ -661,6 +683,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, offsetStart = currentRoundFDStartForMyTargetAgg; } + numBytesPutThisAggRound += bufferAmountToSend; #ifdef onesidedtrace printf("bufferAmountToSend is %d\n",bufferAmountToSend); #endif @@ -848,6 +871,16 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, MPI_Type_free(&targetBufferDerivedDataType); } } + if (!gpfsmpio_onesided_no_rmw) { +#ifndef ACTIVE_TARGET + MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, fd->io_buf_put_amounts_window); +#endif + MPI_Put(&numBytesPutThisAggRound,1, MPI_INT,targetAggsForMyData[aggIter],myrank, 1,MPI_INT,fd->io_buf_put_amounts_window); + +#ifndef ACTIVE_TARGET + MPI_Win_unlock(targetAggsForMyData[aggIter], fd->io_buf_put_amounts_window); +#endif + } } // baseoffset != -1 } // target aggs @@ -855,6 +888,8 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, #ifdef ACTIVE_TARGET MPI_Win_fence(0, write_buf_window); + if (!gpfsmpio_onesided_no_rmw) + MPI_Win_fence(0, fd->io_buf_put_amounts_window); #else MPI_Barrier(fd->comm); #endif @@ -879,20 +914,24 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, printf("currentRoundFDStart is %ld currentRoundFDEnd is %ld within file domeain %ld to %ld\n",currentRoundFDStart,currentRoundFDEnd,fd_start[myAggRank],fd_end[myAggRank]); #endif + int doWriteContig = 1; + if (!gpfsmpio_onesided_no_rmw) { + int numBytesPutIntoBuf = 0; + for (i=0;i<nprocs;i++) { + numBytesPutIntoBuf += write_buf_put_amounts[i]; + write_buf_put_amounts[i] = 0; + } + if (numBytesPutIntoBuf != ((int)(currentRoundFDEnd - currentRoundFDStart)+1)) { + doWriteContig = 0; + *hole_found = 1; + } + } + if (!useIOBuffer) { - ADIO_WriteContig(fd, write_buf, (int)(currentRoundFDEnd - currentRoundFDStart)+1, - MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code); - -/* For now this algorithm cannot handle holes in the source data and does not do any data sieving. - * One possible approach would be to initialize the write buffer with some value and then check to - * see if the mpi_put operations changed the values, if they were unchanged then retry with some other - * default initialized value and if that was still unchanged then you would know where a hole was. - * Here is some initial sample code for that: - * if (roundIter<(numberOfRounds-1)) { - * for (i=0;i<((currentRoundFDEnd - currentRoundFDStart)+1);i++) - * write_buf[i] = '\0'; - * } -*/ + if (doWriteContig) + ADIO_WriteContig(fd, write_buf, (int)(currentRoundFDEnd - currentRoundFDStart)+1, + MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code); + } else { /* use the thread writer */ if(!pthread_equal(io_thread, pthread_self())) { @@ -916,6 +955,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, currentWriteBuf = 0; write_buf = write_buf0; } + if (doWriteContig) { io_thread_args.io_kind = ADIOI_WRITE; io_thread_args.size = (currentRoundFDEnd-currentRoundFDStart) + 1; io_thread_args.offset = currentRoundFDStart; @@ -925,7 +965,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, if ( (pthread_create(&io_thread, NULL, ADIOI_IO_Thread_Func, &(io_thread_args))) != 0) io_thread = pthread_self(); - + } } // useIOBuffer } // iAmUsedAgg @@ -1087,10 +1127,6 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, ADIO_Offset greatestFileDomainOffset = 0; ADIO_Offset smallestFileDomainOffset = lastFileOffset; for (j=0;j<naggs;j++) { - /* Find the actual lowest and highest offsets to be written. - The non-aggs need to know this too to adjust the mpi_get - window displacement accordingly. - */ if (fd_end[j] > greatestFileDomainOffset) { greatestFileDomainOffset = fd_end[j]; greatestFileDomainAggRank = j; @@ -1306,6 +1342,12 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, sourceAggsForMyDataFDStart[numSourceAggs] = firstFileOffset; } sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex]; + /* Round down file domain to the last actual offset used if this is the last file domain. + */ + if (currentAggRankListIndex == greatestFileDomainAggRank) { + if (sourceAggsForMyDataFDEnd[numSourceAggs] > lastFileOffset) + sourceAggsForMyDataFDEnd[numSourceAggs] = lastFileOffset; + } sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; if (bufTypeIsContig) baseRecvBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = currentRecvBufferOffset; @@ -1377,6 +1419,12 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, sourceAggsForMyDataFDStart[numSourceAggs] = firstFileOffset; } sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex]; + /* Round down file domain to the last actual offset used if this is the last file domain. + */ + if (currentAggRankListIndex == greatestFileDomainAggRank) { + if (sourceAggsForMyDataFDEnd[numSourceAggs] > lastFileOffset) + sourceAggsForMyDataFDEnd[numSourceAggs] = lastFileOffset; + } sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; if (bufTypeIsContig) baseRecvBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = currentRecvBufferOffset; @@ -1470,6 +1518,12 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, if (nextRoundFDStart < firstFileOffset) nextRoundFDStart = firstFileOffset; } + else if (myAggRank == greatestFileDomainAggRank) { + if (currentRoundFDEnd > lastFileOffset) + currentRoundFDEnd = lastFileOffset; + if (nextRoundFDEnd > lastFileOffset) + nextRoundFDEnd = lastFileOffset; + } } #ifdef ROMIO_GPFS diff --git a/src/mpi/romio/adio/include/adio.h b/src/mpi/romio/adio/include/adio.h index f1d4ac3..da77015 100644 --- a/src/mpi/romio/adio/include/adio.h +++ b/src/mpi/romio/adio/include/adio.h @@ -237,6 +237,9 @@ typedef struct ADIOI_FileD { int my_cb_nodes_index; /* my index into cb_config_list. -1 if N/A */ char *io_buf; /* two-phase buffer allocated out of i/o path */ MPI_Win io_buf_window; /* Window over the io_buf to support one-sided aggregation */ + int *io_buf_put_amounts; /* array tracking the amount of data mpi_put into the io_buf + during the same round of one-sided write aggregation */ + MPI_Win io_buf_put_amounts_window; /* Window over the io_buf_put_amounts */ /* External32 */ int is_external32; /* bool: 0 means native view */ diff --git a/src/mpi/romio/adio/include/adioi.h b/src/mpi/romio/adio/include/adioi.h index 6320a63..0290cb4 100644 --- a/src/mpi/romio/adio/include/adioi.h +++ b/src/mpi/romio/adio/include/adioi.h @@ -696,7 +696,8 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset *st_offsets, ADIO_Offset *end_offsets, ADIO_Offset *fd_start, - ADIO_Offset* fd_end); + ADIO_Offset* fd_end, + int *hole_found); void ADIOI_OneSidedReadAggregation(ADIO_File fd, ADIO_Offset *offset_list, ADIO_Offset *len_list, http://git.mpich.org/mpich.git/commitdiff/41ab3461e8200920c0fb9a65f8e1921bb… commit 41ab3461e8200920c0fb9a65f8e1921bb428fe87 Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Thu Mar 19 19:05:42 2015 -0500 ROMIO Collective IO One-sided aggregation algorithm avoid read-modify-write for holes at the beginning Added support to correctly handle a data pattern that has a hole only at the beginning of the file offset range to essentially ignore the hole and begin writing at the first offset with actual data, thereby avoiding the need for a read-modify-write. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/common/onesided_aggregation.c b/src/mpi/romio/adio/common/onesided_aggregation.c index 1068fab..50d9315 100644 --- a/src/mpi/romio/adio/common/onesided_aggregation.c +++ b/src/mpi/romio/adio/common/onesided_aggregation.c @@ -119,16 +119,20 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset greatestFileDomainOffset = 0; ADIO_Offset smallestFileDomainOffset = lastFileOffset; for (j=0;j<naggs;j++) { + /* Find the actual lowest and highest offsets to be written. + The non-aggs need to know this too to adjust the mpi_put + window displacement accordingly. + */ + if (fd_end[j] > greatestFileDomainOffset) { + greatestFileDomainOffset = fd_end[j]; + greatestFileDomainAggRank = j; + } + if (fd_start[j] < smallestFileDomainOffset) { + smallestFileDomainOffset = fd_start[j]; + smallestFileDomainAggRank = j; + } if (fd->hints->ranklist[j] == myrank) { myAggRank = j; - if (fd_end[j] > greatestFileDomainOffset) { - greatestFileDomainOffset = fd_end[j]; - greatestFileDomainAggRank = j; - } - if (fd_start[j] < smallestFileDomainOffset) { - smallestFileDomainOffset = fd_start[j]; - smallestFileDomainAggRank = j; - } if (fd_end[j] > fd_start[j]) { iAmUsedAgg = 1; } @@ -333,8 +337,14 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, /* Initialize the data structures if this is the first offset in the round/target agg. */ if (targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] == -1) { - targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex]; + targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex]; targetAggsForMyDataFDStart[numTargetAggs] = fd_start[currentAggRankListIndex]; + /* Round up file domain to the first actual offset used if this is the first file domain. + */ + if (currentAggRankListIndex == smallestFileDomainAggRank) { + if (targetAggsForMyDataFDStart[numTargetAggs] < firstFileOffset) + targetAggsForMyDataFDStart[numTargetAggs] = firstFileOffset; + } targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex]; targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; if (bufTypeIsContig) @@ -398,8 +408,14 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, */ if (blockEnd >= fd_start[currentAggRankListIndex]) { numTargetAggs++; - targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex]; + targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex]; targetAggsForMyDataFDStart[numTargetAggs] = fd_start[currentAggRankListIndex]; + /* Round up file domain to the first actual offset used if this is the first file domain. + */ + if (currentAggRankListIndex == smallestFileDomainAggRank) { + if (targetAggsForMyDataFDStart[numTargetAggs] < firstFileOffset) + targetAggsForMyDataFDStart[numTargetAggs] = firstFileOffset; + } targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex]; targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; if (bufTypeIsContig) @@ -1071,16 +1087,20 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, ADIO_Offset greatestFileDomainOffset = 0; ADIO_Offset smallestFileDomainOffset = lastFileOffset; for (j=0;j<naggs;j++) { + /* Find the actual lowest and highest offsets to be written. + The non-aggs need to know this too to adjust the mpi_get + window displacement accordingly. + */ + if (fd_end[j] > greatestFileDomainOffset) { + greatestFileDomainOffset = fd_end[j]; + greatestFileDomainAggRank = j; + } + if (fd_start[j] < smallestFileDomainOffset) { + smallestFileDomainOffset = fd_start[j]; + smallestFileDomainAggRank = j; + } if (fd->hints->ranklist[j] == myrank) { myAggRank = j; - if (fd_end[j] > greatestFileDomainOffset) { - greatestFileDomainOffset = fd_end[j]; - greatestFileDomainAggRank = j; - } - if (fd_start[j] < smallestFileDomainOffset) { - smallestFileDomainOffset = fd_start[j]; - smallestFileDomainAggRank = j; - } if (fd_end[j] > fd_start[j]) { iAmUsedAgg = 1; } @@ -1277,8 +1297,14 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, /* Initialize the data structures if this is the first offset in the round/source agg. */ if (sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] == -1) { - sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex]; + sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex]; sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[currentAggRankListIndex]; + /* Round up file domain to the first actual offset used if this is the first file domain. + */ + if (currentAggRankListIndex == smallestFileDomainAggRank) { + if (sourceAggsForMyDataFDStart[numSourceAggs] < firstFileOffset) + sourceAggsForMyDataFDStart[numSourceAggs] = firstFileOffset; + } sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex]; sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; if (bufTypeIsContig) @@ -1342,8 +1368,14 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, */ if (blockEnd >= fd_start[currentAggRankListIndex]) { numSourceAggs++; - sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex]; + sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex]; sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[currentAggRankListIndex]; + /* Round up file domain to the first actual offset used if this is the first file domain. + */ + if (currentAggRankListIndex == smallestFileDomainAggRank) { + if (sourceAggsForMyDataFDStart[numSourceAggs] < firstFileOffset) + sourceAggsForMyDataFDStart[numSourceAggs] = firstFileOffset; + } sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex]; sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; if (bufTypeIsContig) http://git.mpich.org/mpich.git/commitdiff/398aeb4fa597ab55aff917066583e5e66… commit 398aeb4fa597ab55aff917066583e5e665631037 Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Wed Mar 11 21:41:04 2015 -0500 ROMIO Collective IO One-sided aggregation algorithm non-contiguous source buffer bug fixes The CESM climate model decomps for fill-value support exposed several bugs in the algorithm related to non-contiguous source buffers which have been fixed. Those issues include: Mishandling of ranks with no data. Miscalculations of the source buffer offsets utilizing the flattened buffer mechanisms. Mishandling of negative source buffer offsets. Inefficient and inaccurate memory management of temporary buffers used to collect non-contigous chunks for a given file offset. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/common/onesided_aggregation.c b/src/mpi/romio/adio/common/onesided_aggregation.c index 695edde..1068fab 100644 --- a/src/mpi/romio/adio/common/onesided_aggregation.c +++ b/src/mpi/romio/adio/common/onesided_aggregation.c @@ -17,7 +17,7 @@ typedef struct NonContigSourceBufOffset { int dataTypeExtent; int flatBufIndice; - ADIO_Offset remLen; + ADIO_Offset indiceOffset; } NonContigSourceBufOffset; void ADIOI_OneSidedWriteAggregation(ADIO_File fd, @@ -74,20 +74,30 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, flatBuf = ADIOI_Flatlist; while (flatBuf->type != datatype) flatBuf = flatBuf->next; MPI_Type_extent(datatype, &bufTypeExtent); +#ifdef onesidedtrace + printf("flatBuf->count is %d bufTypeExtent is %d\n", flatBuf->count,bufTypeExtent); + for (i=0;i<flatBuf->count;i++) + printf("flatBuf->blocklens[%d] is %d flatBuf->indices[%d] is %ld\n",i,flatBuf->blocklens[i],i,flatBuf->indices[i]); +#endif } #ifdef onesidedtrace - printf(" ADIOI_OneSidedWriteAggregation bufTypeIsContig is %d maxNumContigOperations is %d\n",bufTypeIsContig,maxNumContigOperations); + printf(" ADIOI_OneSidedWriteAggregation bufTypeIsContig is %d contig_access_count is %d\n",bufTypeIsContig,contig_access_count); #endif - ADIO_Offset myOffsetStart = st_offsets[myrank], myOffsetEnd = end_offsets[myrank], currentRoundOffsetStart = st_offsets[myrank]; - - ADIO_Offset lastFileOffset = 0, firstFileOffset = myOffsetEnd; + ADIO_Offset lastFileOffset = 0, firstFileOffset = -1; /* Get the total range being written. */ for (j=0;j<nprocs;j++) { - lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]); - firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]); + if (end_offsets[j] > st_offsets[j]) { + /* Guard against ranks with empty data. + */ + lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]); + if (firstFileOffset == -1) + firstFileOffset = st_offsets[j]; + else + firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]); + } } int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */ @@ -195,7 +205,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, int currentDataTypeExtent = 0; int currentFlatBufIndice=0; - ADIO_Offset currentRemLen = 0; + ADIO_Offset currentIndiceOffset = 0; #ifdef onesidedtrace printf("NumberOfRounds is %d\n",numberOfRounds); @@ -206,6 +216,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, #endif int currentAggRankListIndex = 0; + int maxNumNonContigSourceChunks = 0; /* This denotes the coll_bufsize boundaries within the source buffer for writing for the same round. */ @@ -232,31 +243,44 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, /* Non-contiguous source datatype, count up the extents and indices to this point * in the blocks for use in computing the source buffer offset. */ - ADIO_Offset sourceBlockTotal = 0-currentRemLen; + ADIO_Offset sourceBlockTotal = 0; + int lastIndiceUsed = currentFlatBufIndice; + int numNonContigSourceChunks = 0; +#ifdef onesidedtrace + printf("blockIter %d len_list[blockIter-1] is %d currentIndiceOffset is %ld currentFlatBufIndice is %d\n",blockIter,len_list[blockIter-1],currentIndiceOffset,currentFlatBufIndice); +#endif while (sourceBlockTotal < len_list[blockIter-1]) { - maxNumContigOperations++; - sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + numNonContigSourceChunks++; + sourceBlockTotal += (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset); + lastIndiceUsed = currentFlatBufIndice; currentFlatBufIndice++; if (currentFlatBufIndice == flatBuf->count) { currentFlatBufIndice = 0; currentDataTypeExtent++; } + currentIndiceOffset = 0; } if (sourceBlockTotal > len_list[blockIter-1]) { currentFlatBufIndice--; if (currentFlatBufIndice < 0 ) { currentDataTypeExtent--; - currentFlatBufIndice = flatBuf->count; + currentFlatBufIndice = flatBuf->count-1; } + currentIndiceOffset = len_list[blockIter-1] - (sourceBlockTotal - flatBuf->blocklens[lastIndiceUsed]); + ADIOI_Assert((currentIndiceOffset >= 0) && (currentIndiceOffset < flatBuf->blocklens[currentFlatBufIndice])); } - currentRemLen = len_list[blockIter-1] - sourceBlockTotal; + else + currentIndiceOffset = 0; + maxNumContigOperations += numNonContigSourceChunks; + if (numNonContigSourceChunks > maxNumNonContigSourceChunks) + maxNumNonContigSourceChunks = numNonContigSourceChunks; #ifdef onesidedtrace - printf("contig_access_count iter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); + printf("blockiter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentIndiceOffset is now %ld maxNumContigOperations is now %d\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset,maxNumContigOperations); #endif } // !bufTypeIsContig } // blockIter > 0 - /* For the first iteration we need to include these maxNumContigOperations + /* For the first iteration we need to include these maxNumContigOperations and maxNumNonContigSourceChunks * for non-contig case even though we did not need to compute the starting offset. */ if ((blockIter == 0) && (!bufTypeIsContig)) { @@ -264,6 +288,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, int tmpCurrentFlatBufIndice = currentFlatBufIndice; while (sourceBlockTotal < len_list[0]) { maxNumContigOperations++; + maxNumNonContigSourceChunks++; sourceBlockTotal += flatBuf->blocklens[tmpCurrentFlatBufIndice]; tmpCurrentFlatBufIndice++; if (tmpCurrentFlatBufIndice == flatBuf->count) { @@ -272,7 +297,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, } } - ADIO_Offset blockStart = offset_list[blockIter], blockLen = len_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1; + ADIO_Offset blockStart = offset_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1; /* Find the starting target agg for this block - normally it will be the current agg so guard the expensive * while loop with a cheap if-check which for large numbers of small blocks will usually be false. @@ -317,7 +342,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].indiceOffset = currentIndiceOffset; } intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + ((targetAggsForMyDataCurrentRoundIter[numTargetAggs]+1) * coll_bufsize); @@ -349,7 +374,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].indiceOffset = currentIndiceOffset; } targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; @@ -382,7 +407,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].indiceOffset = currentIndiceOffset; } #ifdef onesidedtrace printf("large block init settings numTargetAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld\n",numTargetAggs,i,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter]); @@ -407,7 +432,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].indiceOffset = currentIndiceOffset; } targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; #ifdef onesidedtrace @@ -475,6 +500,16 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, ADIO_Offset currentBaseSourceBufferOffset = 0; + /* These data structures are used to track the offset/len pairs for non-contiguous source buffers that are + * to be used for each block of data in the offset list. Allocate them once with the maximum size and then + * reuse the space throughout the algoritm below. + */ + ADIO_Offset *nonContigSourceOffsets; + int *nonContigSourceLens; + if (!bufTypeIsContig) { + nonContigSourceOffsets = (ADIO_Offset *)ADIOI_Malloc((maxNumNonContigSourceChunks+2) * sizeof(ADIO_Offset)); + nonContigSourceLens = (int *)ADIOI_Malloc((maxNumNonContigSourceChunks+2) * sizeof(int)); + } /* This is the second main loop of the algorithm, actually nested loop of target aggs within rounds. There are 2 flavors of this. * For gpfsmpio_aggmethod of 1 each nested iteration for the target agg does an mpi_put on a contiguous chunk using a primative datatype * determined using the data structures from the first main loop. For gpfsmpio_aggmethod of 2 each nested iteration for the target agg @@ -513,7 +548,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, */ int offsetIter; int startingOffLenIndex = targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter], endingOffLenIndex = targetAggsForMyDataLastOffLenIndex[roundIter][aggIter]; - for (offsetIter=startingOffLenIndex;offsetIter<=endingOffLenIndex;offsetIter++) { + for (offsetIter=startingOffLenIndex;offsetIter<=endingOffLenIndex;offsetIter++) { if (currentRoundFDEndForMyTargetAgg > targetAggsForMyDataFDEnd[aggIter]) currentRoundFDEndForMyTargetAgg = targetAggsForMyDataFDEnd[aggIter]; @@ -527,7 +562,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, if (bufTypeIsContig) currentBaseSourceBufferOffset = baseSourceBufferOffset[roundIter][aggIter]; else { - currentRemLen = baseNonContigSourceBufferOffset[roundIter][aggIter].remLen; + currentIndiceOffset = baseNonContigSourceBufferOffset[roundIter][aggIter].indiceOffset; currentDataTypeExtent = baseNonContigSourceBufferOffset[roundIter][aggIter].dataTypeExtent; currentFlatBufIndice = baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice; } @@ -537,22 +572,35 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, currentBaseSourceBufferOffset += len_list[offsetIter-1]; else { - /* For non-contiguous source datatype count up the extents and indices to this point. + /* For non-contiguous source datatype advance the flattened buffer machinery to this offset. + * Note that currentDataTypeExtent, currentFlatBufIndice and currentIndiceOffset are used and + * advanced across the offsetIters. */ - ADIO_Offset sourceBlockTotal = 0-currentRemLen; + ADIO_Offset sourceBlockTotal = 0; + int lastIndiceUsed = currentFlatBufIndice; while (sourceBlockTotal < len_list[offsetIter-1]) { - sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + sourceBlockTotal += (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset); + lastIndiceUsed = currentFlatBufIndice; currentFlatBufIndice++; if (currentFlatBufIndice == flatBuf->count) { currentFlatBufIndice = 0; currentDataTypeExtent++; } + currentIndiceOffset = 0; } // while - if (sourceBlockTotal > len_list[offsetIter-1]) + if (sourceBlockTotal > len_list[offsetIter-1]) { currentFlatBufIndice--; - currentRemLen = len_list[offsetIter-1] - sourceBlockTotal; + if (currentFlatBufIndice < 0 ) { + currentDataTypeExtent--; + currentFlatBufIndice = flatBuf->count-1; + } + currentIndiceOffset = len_list[offsetIter-1] - (sourceBlockTotal - flatBuf->blocklens[lastIndiceUsed]); + ADIOI_Assert((currentIndiceOffset >= 0) && (currentIndiceOffset < flatBuf->blocklens[currentFlatBufIndice])); + } + else + currentIndiceOffset = 0; #ifdef onesidedtrace - printf("contig_access_count target agg %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",aggIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); + printf("offsetIter %d contig_access_count target agg %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentIndiceOffset is now %ld\n",offsetIter,aggIter,currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset); #endif } } @@ -597,6 +645,9 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, offsetStart = currentRoundFDStartForMyTargetAgg; } +#ifdef onesidedtrace + printf("bufferAmountToSend is %d\n",bufferAmountToSend); +#endif if (bufferAmountToSend > 0) { /* we have data to send this round */ if (gpfsmpio_aggmethod == 2) { /* Only allocate these arrays if we are using method 2 and only do it once for this round/target agg. @@ -613,73 +664,55 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, * from the source buffer to be sent to this contiguous chunk defined by the round/agg/iter in the target. */ int numNonContigSourceChunks = 0; - ADIO_Offset *nonContigSourceOffsets; - int *nonContigSourceLens; ADIO_Offset baseDatatypeInstanceOffset = 0; if (!bufTypeIsContig) { - currentSourceBufferOffset = (ADIO_Offset)((ADIO_Offset)currentDataTypeExtent * (ADIO_Offset)bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentRemLen; + currentSourceBufferOffset = (ADIO_Offset)((ADIO_Offset)currentDataTypeExtent * (ADIO_Offset)bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentIndiceOffset; #ifdef onesidedtrace - printf("!bufTypeIsContig currentSourceBufferOffset set to %ld for roundIter %d target %d currentDataTypeExtent %d flatBuf->indices[currentFlatBufIndice] %ld currentRemLen %ld currentFlatBufIndice %d\n",currentSourceBufferOffset,roundIter,aggIter,currentDataTypeExtent,flatBuf->indices[currentFlatBufIndice],currentRemLen,currentFlatBufIndice); + printf("!bufTypeIsContig currentSourceBufferOffset set to %ld for roundIter %d target %d currentDataTypeExtent %d flatBuf->indices[currentFlatBufIndice] %ld currentIndiceOffset %ld currentFlatBufIndice %d\n",currentSourceBufferOffset,roundIter,aggIter,currentDataTypeExtent,flatBuf->indices[currentFlatBufIndice],currentIndiceOffset,currentFlatBufIndice); #endif - /* Count the chunks first to see how much to malloc from the ending point from above code. + /* Use a tmp variable for the currentFlatBufIndice and currentIndiceOffset as they are used across the offsetIters + * to compute the starting point for this iteration and will be modified now to compute the data chunks for + * this iteration. */ - int sendBytesCounted = 0; int tmpFlatBufIndice = currentFlatBufIndice; - int maxNumNonContigSourceChunks = 2; // over-initialize for potential remnants on both ends - - while (sendBytesCounted < bufferAmountToSend) { - maxNumNonContigSourceChunks++; - if (tmpFlatBufIndice == flatBuf->count) { - tmpFlatBufIndice = 0; - } - sendBytesCounted += flatBuf->blocklens[tmpFlatBufIndice]; - tmpFlatBufIndice++; - } - - nonContigSourceOffsets = (ADIO_Offset *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(ADIO_Offset)); - nonContigSourceLens = (int *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(int)); + ADIO_Offset tmpIndiceOffset = currentIndiceOffset; /* now populate the nonContigSourceOffsets and nonContigSourceLens arrays for use in the one-sided operations. */ int ncArrayIndex = 0; int remainingBytesToLoadedIntoNCArrays = bufferAmountToSend; - ADIO_Offset indexIntoCurrentIndice = 0; - if (currentRemLen > 0) - indexIntoCurrentIndice = flatBuf->blocklens[currentFlatBufIndice] - currentRemLen; - int datatypeInstances = currentDataTypeExtent; - tmpFlatBufIndice = currentFlatBufIndice; while (remainingBytesToLoadedIntoNCArrays > 0) { - nonContigSourceOffsets[ncArrayIndex] = currentSourceBufferOffset; + ADIOI_Assert(ncArrayIndex < (maxNumNonContigSourceChunks+2)); + nonContigSourceOffsets[ncArrayIndex] = currentSourceBufferOffset; - if ((flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice) > remainingBytesToLoadedIntoNCArrays) { + if ((flatBuf->blocklens[tmpFlatBufIndice] - tmpIndiceOffset) >= remainingBytesToLoadedIntoNCArrays) { nonContigSourceLens[ncArrayIndex] = remainingBytesToLoadedIntoNCArrays; remainingBytesToLoadedIntoNCArrays = 0; } else { - nonContigSourceLens[ncArrayIndex] = (int)(flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); - remainingBytesToLoadedIntoNCArrays -= (flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); - } - indexIntoCurrentIndice = 0; // only worry about beginning remnant for first iter - - tmpFlatBufIndice++; - if (tmpFlatBufIndice == flatBuf->count) { - tmpFlatBufIndice = 0; - datatypeInstances++; - baseDatatypeInstanceOffset = datatypeInstances * bufTypeExtent; + nonContigSourceLens[ncArrayIndex] = (int)(flatBuf->blocklens[tmpFlatBufIndice] - tmpIndiceOffset); + remainingBytesToLoadedIntoNCArrays -= (flatBuf->blocklens[tmpFlatBufIndice] - tmpIndiceOffset); + tmpIndiceOffset = 0; + tmpFlatBufIndice++; + if (tmpFlatBufIndice == flatBuf->count) { + tmpFlatBufIndice = 0; + datatypeInstances++; + baseDatatypeInstanceOffset = datatypeInstances * bufTypeExtent; + } + currentSourceBufferOffset = baseDatatypeInstanceOffset + flatBuf->indices[tmpFlatBufIndice]; } - currentSourceBufferOffset = baseDatatypeInstanceOffset + flatBuf->indices[tmpFlatBufIndice]; +#ifdef onesidedtrace + printf("currentSourceBufferOffset set to %ld off of baseDatatypeInstanceOffset of %ld + tmpFlatBufIndice %d with value %ld\n ncArrayIndex is %d nonContigSourceOffsets[ncArrayIndex] is %ld nonContigSourceLens[ncArrayIndex] is %ld\n",currentSourceBufferOffset,baseDatatypeInstanceOffset, tmpFlatBufIndice, flatBuf->indices[tmpFlatBufIndice],ncArrayIndex,nonContigSourceOffsets[ncArrayIndex],nonContigSourceLens[ncArrayIndex]); +#endif ncArrayIndex++; numNonContigSourceChunks++; } // while -#ifdef onesidedtrace - printf("CurrentSourceBufferOffset finally set to %ld\n",currentSourceBufferOffset); -#endif - } // !bufTypeIsContig + } // !bufTypeIsContig /* Determine the offset into the target window. */ @@ -698,13 +731,14 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, write_buf_window); #endif if (bufTypeIsContig) { - MPI_Put(&((char*)buf)[sourceBufferOffset],bufferAmountToSend, MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, bufferAmountToSend,MPI_BYTE,write_buf_window); + MPI_Put(((char*)buf) + sourceBufferOffset,bufferAmountToSend, MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, bufferAmountToSend,MPI_BYTE,write_buf_window); } else { for (i=0;i<numNonContigSourceChunks;i++) { - MPI_Put(&((char*)buf)[nonContigSourceOffsets[i]],nonContigSourceLens[i], MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, nonContigSourceLens[i],MPI_BYTE,write_buf_window); + MPI_Put(((char*)buf) + nonContigSourceOffsets[i],nonContigSourceLens[i], MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, nonContigSourceLens[i],MPI_BYTE,write_buf_window); + #ifdef onesidedtrace - printf("mpi_put[%d] nonContigSourceOffsets is %d of nonContigSourceLens %d to target disp %d\n",i,nonContigSourceOffsets[i],nonContigSourceLens[i],targetDisplacementToUseThisRound); + printf("mpi_put[%d] nonContigSourceOffsets is %d of nonContigSourceLens %d to target disp %d first int of data: %d\n ",i,nonContigSourceOffsets[i],nonContigSourceLens[i],targetDisplacementToUseThisRound, ((int*)(((char*)buf) + nonContigSourceOffsets[i]))[0]); #endif targetDisplacementToUseThisRound += nonContigSourceLens[i]; } @@ -729,6 +763,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, } else { for (i=0;i<numNonContigSourceChunks;i++) { + if (nonContigSourceLens[i] > 0) { targetAggBlockLengths[targetAggContigAccessCount]= nonContigSourceLens[i]; targetAggDataTypes[targetAggContigAccessCount] = MPI_BYTE; targetAggDisplacements[targetAggContigAccessCount] = targetDisplacementToUseThisRound; @@ -739,16 +774,13 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, targetAggContigAccessCount++; targetDisplacementToUseThisRound += nonContigSourceLens[i]; } + } } } #ifdef onesidedtrace printf("roundIter %d bufferAmountToSend is %d sourceBufferOffset is %d offsetStart is %ld currentRoundFDStartForMyTargetAgg is %ld targetDisplacementToUseThisRound is %ld targetAggsForMyDataFDStart[aggIter] is %ld\n",roundIter, bufferAmountToSend,sourceBufferOffset, offsetStart,currentRoundFDStartForMyTargetAgg,targetDisplacementToUseThisRound,targetAggsForMyDataFDStart[aggIter]); #endif - if (!bufTypeIsContig) { - ADIOI_Free(nonContigSourceOffsets); - ADIOI_Free(nonContigSourceLens); - } } // bufferAmountToSend > 0 } // contig list @@ -756,6 +788,20 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, */ if (gpfsmpio_aggmethod == 2) { MPI_Datatype sourceBufferDerivedDataType, targetBufferDerivedDataType; + if (targetAggContigAccessCount > 0) { + /* Rebase source buffer offsets to 0 if there are any negative offsets for safety + * when iteracting with PAMI. + */ + MPI_Aint lowestDisplacement = 0; + for (i=0;i<targetAggContigAccessCount;i++) { + if (sourceBufferDisplacements[i] < lowestDisplacement) + lowestDisplacement = sourceBufferDisplacements[i]; + } + if (lowestDisplacement < 0) { + lowestDisplacement *= -1; + for (i=0;i<targetAggContigAccessCount;i++) + sourceBufferDisplacements[i] += lowestDisplacement; + } MPI_Type_create_struct(targetAggContigAccessCount, targetAggBlockLengths, sourceBufferDisplacements, targetAggDataTypes, &sourceBufferDerivedDataType); MPI_Type_commit(&sourceBufferDerivedDataType); MPI_Type_create_struct(targetAggContigAccessCount, targetAggBlockLengths, targetAggDisplacements, targetAggDataTypes, &targetBufferDerivedDataType); @@ -768,20 +814,23 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, write_buf_window); #endif - MPI_Put(((char*)buf),1, sourceBufferDerivedDataType,targetAggsForMyData[aggIter],0, 1,targetBufferDerivedDataType,write_buf_window); + MPI_Put((((char*)buf) - lowestDisplacement),1, sourceBufferDerivedDataType,targetAggsForMyData[aggIter],0, 1,targetBufferDerivedDataType,write_buf_window); #ifndef ACTIVE_TARGET MPI_Win_unlock(targetAggsForMyData[aggIter], write_buf_window); #endif + } if (allocatedDerivedTypeArrays) { ADIOI_Free(targetAggBlockLengths); ADIOI_Free(targetAggDisplacements); ADIOI_Free(targetAggDataTypes); ADIOI_Free(sourceBufferDisplacements); } + if (targetAggContigAccessCount > 0) { MPI_Type_free(&sourceBufferDerivedDataType); MPI_Type_free(&targetBufferDerivedDataType); + } } } // baseoffset != -1 } // target aggs @@ -817,7 +866,6 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, if (!useIOBuffer) { ADIO_WriteContig(fd, write_buf, (int)(currentRoundFDEnd - currentRoundFDStart)+1, MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code); - int numBytesWritten= (int)(currentRoundFDEnd - currentRoundFDStart)+1; /* For now this algorithm cannot handle holes in the source data and does not do any data sieving. * One possible approach would be to initialize the write buffer with some value and then check to @@ -825,7 +873,7 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, * default initialized value and if that was still unchanged then you would know where a hole was. * Here is some initial sample code for that: * if (roundIter<(numberOfRounds-1)) { - * for (i=0;i<numBytesWritten;i++) + * for (i=0;i<((currentRoundFDEnd - currentRoundFDStart)+1);i++) * write_buf[i] = '\0'; * } */ @@ -885,6 +933,11 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, } /* for-loop roundIter */ + if (!bufTypeIsContig) { + ADIOI_Free(nonContigSourceOffsets); + ADIOI_Free(nonContigSourceLens); + } + #ifdef ROMIO_GPFS endTimeBase = MPI_Wtime(); gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase); @@ -918,6 +971,8 @@ void ADIOI_OneSidedWriteAggregation(ADIO_File fd, else ADIOI_Free(baseNonContigSourceBufferOffset); + if (!bufTypeIsContig) + ADIOI_Delete_flattened(datatype); return; } @@ -978,18 +1033,23 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, MPI_Type_extent(datatype, &bufTypeExtent); } #ifdef onesidedtrace - printf("ADIOI_OneSidedReadAggregation bufTypeIsContig is %d maxNumContigOperations is %d\n",bufTypeIsContig,maxNumContigOperations); + printf("ADIOI_OneSidedReadAggregation bufTypeIsContig is %d contig_access_count is %d\n",bufTypeIsContig,contig_access_count); #endif - ADIO_Offset myOffsetStart = st_offsets[myrank], myOffsetEnd = end_offsets[myrank], currentRoundOffsetStart = st_offsets[myrank]; - - ADIO_Offset lastFileOffset = 0, firstFileOffset = myOffsetEnd; + ADIO_Offset lastFileOffset = 0, firstFileOffset = -1; /* Get the total range being read. */ for (j=0;j<nprocs;j++) { - lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]); - firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]); + if (end_offsets[j] > st_offsets[j]) { + /* Guard against ranks with empty data. + */ + lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]); + if (firstFileOffset == -1) + firstFileOffset = st_offsets[j]; + else + firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]); + } } int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */ @@ -1095,7 +1155,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, int currentDataTypeExtent = 0; int currentFlatBufIndice=0; - ADIO_Offset currentRemLen = 0; + ADIO_Offset currentIndiceOffset = 0; #ifdef onesidedtrace printf("NumberOfRounds is %d\n",numberOfRounds); @@ -1106,6 +1166,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, #endif int currentAggRankListIndex = 0; + int maxNumNonContigSourceChunks = 0; /* This denotes the coll_bufsize boundaries within the source buffer for reading for 1 round. */ @@ -1133,31 +1194,41 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, /* Non-contiguous source datatype, count up the extents and indices to this point * in the blocks. */ - ADIO_Offset sourceBlockTotal = 0-currentRemLen; + ADIO_Offset sourceBlockTotal = 0; + int lastIndiceUsed; + int numNonContigSourceChunks = 0; while (sourceBlockTotal < len_list[blockIter-1]) { - maxNumContigOperations++; - sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + numNonContigSourceChunks++; + sourceBlockTotal += (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset); + lastIndiceUsed = currentFlatBufIndice; currentFlatBufIndice++; if (currentFlatBufIndice == flatBuf->count) { currentFlatBufIndice = 0; currentDataTypeExtent++; } + currentIndiceOffset = 0; } if (sourceBlockTotal > len_list[blockIter-1]) { currentFlatBufIndice--; if (currentFlatBufIndice < 0 ) { currentDataTypeExtent--; - currentFlatBufIndice = flatBuf->count; + currentFlatBufIndice = flatBuf->count-1; } + currentIndiceOffset = len_list[blockIter-1] - (sourceBlockTotal - flatBuf->blocklens[lastIndiceUsed]); + ADIOI_Assert((currentIndiceOffset >= 0) && (currentIndiceOffset < flatBuf->blocklens[currentFlatBufIndice])); } - currentRemLen = len_list[blockIter-1] - sourceBlockTotal; + else + currentIndiceOffset = 0; + maxNumContigOperations += numNonContigSourceChunks; + if (numNonContigSourceChunks > maxNumNonContigSourceChunks) + maxNumNonContigSourceChunks = numNonContigSourceChunks; #ifdef onesidedtrace - printf("contig_access_count iter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); + printf("block iter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentIndiceOffset is now %ld maxNumContigOperations is now %d\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset,maxNumContigOperations); #endif } // !bufTypeIsContig } // blockIter > 0 - /* For the first iteration we need to include these maxNumContigOperations + /* For the first iteration we need to include these maxNumContigOperations and maxNumNonContigSourceChunks * for non-contig case even though we did not need to compute the starting offset. */ if ((blockIter == 0) && (!bufTypeIsContig)) { @@ -1165,6 +1236,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, int tmpCurrentFlatBufIndice = currentFlatBufIndice; while (sourceBlockTotal < len_list[0]) { maxNumContigOperations++; + maxNumNonContigSourceChunks++; sourceBlockTotal += flatBuf->blocklens[tmpCurrentFlatBufIndice]; tmpCurrentFlatBufIndice++; if (tmpCurrentFlatBufIndice == flatBuf->count) { @@ -1173,7 +1245,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, } } - ADIO_Offset blockStart = offset_list[blockIter], blockLen = len_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1; + ADIO_Offset blockStart = offset_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1; /* Find the starting source agg for this block - normally it will be the current agg so guard the expensive * while loop with a cheap if-check which for large numbers of small blocks will usually be false. @@ -1214,7 +1286,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].indiceOffset = currentIndiceOffset; } intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + ((sourceAggsForMyDataCurrentRoundIter[numSourceAggs]+1) * coll_bufsize); @@ -1246,7 +1318,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].indiceOffset = currentIndiceOffset; } sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; @@ -1279,7 +1351,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].indiceOffset = currentIndiceOffset; } #ifdef onesidedtrace @@ -1305,7 +1377,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, else { baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; - baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].indiceOffset = currentIndiceOffset; } sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; #ifdef onesidedtrace @@ -1377,6 +1449,16 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, ADIO_Offset currentBaseRecvBufferOffset = 0; + /* These data structures are used to track the offset/len pairs for non-contiguous source buffers that are + * to be used for each block of data in the offset list. Allocate them once with the maximum size and then + * reuse the space throughout the algoritm below. + */ + ADIO_Offset *nonContigSourceOffsets; + int *nonContigSourceLens; + if (!bufTypeIsContig) { + nonContigSourceOffsets = (ADIO_Offset *)ADIOI_Malloc((maxNumNonContigSourceChunks+2) * sizeof(ADIO_Offset)); + nonContigSourceLens = (int *)ADIOI_Malloc((maxNumNonContigSourceChunks+2) * sizeof(int)); + } /* This is the second main loop of the algorithm, actually nested loop of source aggs within rounds. There are 2 flavors of this. * For gpfsmpio_aggmethod of 1 each nested iteration for the source agg does an mpi_put on a contiguous chunk using a primative datatype * determined using the data structures from the first main loop. For gpfsmpio_aggmethod of 2 each nested iteration for the source agg @@ -1542,7 +1624,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, else { currentFlatBufIndice = baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice; currentDataTypeExtent = baseNonContigSourceBufferOffset[roundIter][aggIter].dataTypeExtent; - currentRemLen = baseNonContigSourceBufferOffset[roundIter][aggIter].remLen; + currentIndiceOffset = baseNonContigSourceBufferOffset[roundIter][aggIter].indiceOffset; #ifdef onesidedtrace printf("currentFlatBufIndice initially set to %d starting this round/agg %d/%d\n",currentFlatBufIndice,roundIter,aggIter); #endif @@ -1553,22 +1635,35 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, currentBaseRecvBufferOffset += len_list[offsetIter-1]; else { - /* For non-contiguous source datatype count up the extents and indices to this point. + /* For non-contiguous source datatype advance the flattened buffer machinery to this offset. + * Note that currentDataTypeExtent, currentFlatBufIndice and currentIndiceOffset are used and + * advanced across the offsetIters. */ - ADIO_Offset sourceBlockTotal = 0-currentRemLen; + ADIO_Offset sourceBlockTotal = 0; + int lastIndiceUsed; while (sourceBlockTotal < len_list[offsetIter-1]) { - sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + sourceBlockTotal += (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset); + lastIndiceUsed = currentFlatBufIndice; currentFlatBufIndice++; if (currentFlatBufIndice == flatBuf->count) { currentFlatBufIndice = 0; currentDataTypeExtent++; } + currentIndiceOffset = 0; } // while - if (sourceBlockTotal > len_list[offsetIter-1]) + if (sourceBlockTotal > len_list[offsetIter-1]) { currentFlatBufIndice--; - currentRemLen = len_list[offsetIter-1] - sourceBlockTotal; + if (currentFlatBufIndice < 0 ) { + currentDataTypeExtent--; + currentFlatBufIndice = flatBuf->count-1; + } + currentIndiceOffset = len_list[offsetIter-1] - (sourceBlockTotal - flatBuf->blocklens[lastIndiceUsed]); + ADIOI_Assert((currentIndiceOffset >= 0) && (currentIndiceOffset < flatBuf->blocklens[currentFlatBufIndice])); + } + else + currentIndiceOffset = 0; #ifdef onesidedtrace - printf("contig_access_count source agg %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",aggIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); + printf("contig_access_count source agg %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentIndiceOffset is now %ld\n",aggIter,currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset); #endif } @@ -1632,65 +1727,48 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, * from the source buffer to be sent to this contiguous chunk defined by the round/agg/iter in the source. */ int numNonContigSourceChunks = 0; - ADIO_Offset *nonContigSourceOffsets; - int *nonContigSourceLens; ADIO_Offset baseDatatypeInstanceOffset = 0; if (!bufTypeIsContig) { - currentRecvBufferOffset = (ADIO_Offset)((ADIO_Offset)currentDataTypeExtent * (ADIO_Offset)bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentRemLen; + currentRecvBufferOffset = (ADIO_Offset)((ADIO_Offset)currentDataTypeExtent * (ADIO_Offset)bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentIndiceOffset; #ifdef onesidedtrace - printf("!bufTypeIsContig currentRecvBufferOffset set to %ld for roundIter %d source %d currentDataTypeExtent %d flatBuf->indices[currentFlatBufIndice] %ld currentRemLen %ld currentFlatBufIndice %d\n",currentRecvBufferOffset,roundIter,aggIter,currentDataTypeExtent,flatBuf->indices[currentFlatBufIndice],currentRemLen,currentFlatBufIndice); + printf("!bufTypeIsContig currentRecvBufferOffset set to %ld for roundIter %d source %d currentDataTypeExtent %d flatBuf->indices[currentFlatBufIndice] %ld currentIndiceOffset %ld currentFlatBufIndice %d\n",currentRecvBufferOffset,roundIter,aggIter,currentDataTypeExtent,flatBuf->indices[currentFlatBufIndice],currentIndiceOffset,currentFlatBufIndice); #endif - /* Count the chunks first to see how much to malloc from the ending point from above code. + /* Use a tmp variable for the currentFlatBufIndice and currentIndiceOffset as they are used across the offsetIters + * to compute the starting point for this iteration and will be modified now to compute the data chunks for + * this iteration. */ - int recvBytesCounted = 0; int tmpFlatBufIndice = currentFlatBufIndice; - int maxNumNonContigSourceChunks = 2; // over-initialize for potential remnants on both ends - - while (recvBytesCounted < bufferAmountToRecv) { - maxNumNonContigSourceChunks++; - if (tmpFlatBufIndice == flatBuf->count) { - tmpFlatBufIndice = 0; - } - recvBytesCounted += flatBuf->blocklens[tmpFlatBufIndice]; - tmpFlatBufIndice++; - } - - nonContigSourceOffsets = (ADIO_Offset *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(ADIO_Offset)); - nonContigSourceLens = (int *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(int)); + ADIO_Offset tmpIndiceOffset = currentIndiceOffset; /* now populate the nonContigSourceOffsets and nonContigSourceLens arrays for use in the one-sided operations. */ int ncArrayIndex = 0; int remainingBytesToLoadedIntoNCArrays = bufferAmountToRecv; - ADIO_Offset indexIntoCurrentIndice = 0; - if (currentRemLen > 0) - indexIntoCurrentIndice = flatBuf->blocklens[currentFlatBufIndice] - currentRemLen; int datatypeInstances = currentDataTypeExtent; - tmpFlatBufIndice = currentFlatBufIndice; while (remainingBytesToLoadedIntoNCArrays > 0) { - nonContigSourceOffsets[ncArrayIndex] = currentRecvBufferOffset; + ADIOI_Assert(ncArrayIndex < (maxNumNonContigSourceChunks+2)); + nonContigSourceOffsets[ncArrayIndex] = currentRecvBufferOffset; - if ((flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice) > remainingBytesToLoadedIntoNCArrays) { + if ((flatBuf->blocklens[tmpFlatBufIndice] - tmpIndiceOffset) >= remainingBytesToLoadedIntoNCArrays) { nonContigSourceLens[ncArrayIndex] = remainingBytesToLoadedIntoNCArrays; remainingBytesToLoadedIntoNCArrays = 0; } else { - nonContigSourceLens[ncArrayIndex] = (int)(flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); - remainingBytesToLoadedIntoNCArrays -= (flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); - } - indexIntoCurrentIndice = 0; // only worry about beginning remnant for first iter - - tmpFlatBufIndice++; - if (tmpFlatBufIndice == flatBuf->count) { - tmpFlatBufIndice = 0; - datatypeInstances++; - baseDatatypeInstanceOffset = datatypeInstances * bufTypeExtent; + nonContigSourceLens[ncArrayIndex] = (int)(flatBuf->blocklens[tmpFlatBufIndice] - tmpIndiceOffset); + remainingBytesToLoadedIntoNCArrays -= (flatBuf->blocklens[tmpFlatBufIndice] - tmpIndiceOffset); + tmpIndiceOffset = 0; + tmpFlatBufIndice++; + if (tmpFlatBufIndice == flatBuf->count) { + tmpFlatBufIndice = 0; + datatypeInstances++; + baseDatatypeInstanceOffset = datatypeInstances * bufTypeExtent; + } + currentRecvBufferOffset = baseDatatypeInstanceOffset + flatBuf->indices[tmpFlatBufIndice]; } - currentRecvBufferOffset = baseDatatypeInstanceOffset + flatBuf->indices[tmpFlatBufIndice]; ncArrayIndex++; numNonContigSourceChunks++; } // while @@ -1703,7 +1781,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, */ MPI_Aint sourceDisplacementToUseThisRound = (MPI_Aint) ((ADIO_Offset)offsetStart - currentRoundFDStartForMySourceAgg); - /* If using the thread readr select the appropriate side of the split window. + /* If using the thread reader select the appropriate side of the split window. */ if (useIOBuffer && (read_buf == read_buf1)) { sourceDisplacementToUseThisRound += coll_bufsize; @@ -1716,11 +1794,11 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, MPI_Win_lock(MPI_LOCK_SHARED, sourceAggsForMyData[aggIter], 0, read_buf_window); #endif if (bufTypeIsContig) { - MPI_Get(&((char*)buf)[recvBufferOffset],bufferAmountToRecv, MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, bufferAmountToRecv,MPI_BYTE,read_buf_window); + MPI_Get(((char*)buf) + recvBufferOffset,bufferAmountToRecv, MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, bufferAmountToRecv,MPI_BYTE,read_buf_window); } else { for (i=0;i<numNonContigSourceChunks;i++) { - MPI_Get(&((char*)buf)[nonContigSourceOffsets[i]],nonContigSourceLens[i], MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, nonContigSourceLens[i],MPI_BYTE,read_buf_window); + MPI_Get(((char*)buf) + nonContigSourceOffsets[i],nonContigSourceLens[i], MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, nonContigSourceLens[i],MPI_BYTE,read_buf_window); #ifdef onesidedtrace printf("mpi_put[%d] nonContigSourceOffsets is %d of nonContigSourceLens %d to source disp %d\n",i,nonContigSourceOffsets[i],nonContigSourceLens[i],sourceDisplacementToUseThisRound); #endif @@ -1764,10 +1842,6 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, printf("roundIter %d bufferAmountToRecv is %d recvBufferOffset is %d offsetStart is %ld currentRoundFDStartForMySourceAgg is %ld sourceDisplacementToUseThisRound is %ld sourceAggsForMyDataFDStart[aggIter] is %ld\n",roundIter, bufferAmountToRecv,recvBufferOffset, offsetStart,currentRoundFDStartForMySourceAgg,sourceDisplacementToUseThisRound,sourceAggsForMyDataFDStart[aggIter]); #endif - if (!bufTypeIsContig) { - ADIOI_Free(nonContigSourceOffsets); - ADIOI_Free(nonContigSourceLens); - } } // bufferAmountToRecv > 0 } // contig list @@ -1775,19 +1849,33 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, */ if (gpfsmpio_aggmethod == 2) { MPI_Datatype recvBufferDerivedDataType, sourceBufferDerivedDataType; + if (sourceAggContigAccessCount > 0) { + /* Rebase source buffer offsets to 0 if there are any negative offsets for safety + * when iteracting with PAMI. + */ + MPI_Aint lowestDisplacement = 0; + for (i=0;i<sourceAggContigAccessCount;i++) { + if (recvBufferDisplacements[i] < lowestDisplacement) + lowestDisplacement = recvBufferDisplacements[i]; + } + if (lowestDisplacement < 0) { + lowestDisplacement *= -1; + for (i=0;i<sourceAggContigAccessCount;i++) + recvBufferDisplacements[i] += lowestDisplacement; + } MPI_Type_create_struct(sourceAggContigAccessCount, sourceAggBlockLengths, recvBufferDisplacements, sourceAggDataTypes, &recvBufferDerivedDataType); MPI_Type_commit(&recvBufferDerivedDataType); MPI_Type_create_struct(sourceAggContigAccessCount, sourceAggBlockLengths, sourceAggDisplacements, sourceAggDataTypes, &sourceBufferDerivedDataType); MPI_Type_commit(&sourceBufferDerivedDataType); -// printf("round %d mpi_put of derived type to agg %d sourceAggContigAccessCount is %d\n",roundIter, sourceAggsForMyData[aggIter],sourceAggContigAccessCount); #ifndef ACTIVE_TARGET MPI_Win_lock(MPI_LOCK_SHARED, sourceAggsForMyData[aggIter], 0, read_buf_window); #endif - MPI_Get(((char*)buf),1, recvBufferDerivedDataType,sourceAggsForMyData[aggIter],0, 1,sourceBufferDerivedDataType,read_buf_window); + MPI_Get((((char*)buf) - lowestDisplacement),1, recvBufferDerivedDataType,sourceAggsForMyData[aggIter],0, 1,sourceBufferDerivedDataType,read_buf_window); #ifndef ACTIVE_TARGET MPI_Win_unlock(sourceAggsForMyData[aggIter], read_buf_window); #endif + } if (allocatedDerivedTypeArrays) { ADIOI_Free(sourceAggBlockLengths); @@ -1795,8 +1883,10 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, ADIOI_Free(sourceAggDataTypes); ADIOI_Free(recvBufferDisplacements); } + if (sourceAggContigAccessCount > 0) { MPI_Type_free(&recvBufferDerivedDataType); MPI_Type_free(&sourceBufferDerivedDataType); + } } } // baseoffset != -1 } // source aggs @@ -1813,6 +1903,10 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, } /* for-loop roundIter */ + if (!bufTypeIsContig) { + ADIOI_Free(nonContigSourceOffsets); + ADIOI_Free(nonContigSourceLens); + } #ifdef ROMIO_GPFS endTimeBase = MPI_Wtime(); gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase); @@ -1846,5 +1940,7 @@ void ADIOI_OneSidedReadAggregation(ADIO_File fd, else ADIOI_Free(baseNonContigSourceBufferOffset); + if (!bufTypeIsContig) + ADIOI_Delete_flattened(datatype); return; } http://git.mpich.org/mpich.git/commitdiff/e6849ac4f0a85a58afefc7a0ee9643dbf… commit e6849ac4f0a85a58afefc7a0ee9643dbf225852f Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Sat Feb 28 00:59:54 2015 +0000 ROMIO Collective IO One-sided aggregation algorithm GPFS enablement Code to enable the usage of the optimized one-sided collective IO aggregation algorithm from the ADIOI_GPFS_WriteStridedColl and ADIOI_GPFS_ReadStridedColl functions. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c index c2cad8b..18d92d1 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c @@ -271,6 +271,21 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count, fd->hints->striping_unit); GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART ); + if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { + /* If the user has specified to use a one-sided aggregation method then do that at + * this point instead of the two-phase I/O. + */ + ADIOI_OneSidedReadAggregation(fd, offset_list, len_list, contig_access_count, buf, + datatype,error_code, st_offsets, end_offsets, fd_start, fd_end); + GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs) + ADIOI_Free(offset_list); + ADIOI_Free(len_list); + ADIOI_Free(st_offsets); + ADIOI_Free(end_offsets); + ADIOI_Free(fd_start); + ADIOI_Free(fd_end); + goto fn_exit; + } if (gpfsmpio_p2pcontig==1) { /* For some simple yet common(?) workloads, full-on two-phase I/O is * overkill. We can establish sub-groups of processes and their diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 968e6e6..9ce68fa 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -263,6 +263,20 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count, GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART ); + if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { + /* If the user has specified to use a one-sided aggregation method then do that at + * this point instead of the two-phase I/O. + */ + ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count, buf, datatype, error_code, st_offsets, end_offsets, fd_start, fd_end); + GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs) + ADIOI_Free(offset_list); + ADIOI_Free(len_list); + ADIOI_Free(st_offsets); + ADIOI_Free(end_offsets); + ADIOI_Free(fd_start); + ADIOI_Free(fd_end); + goto fn_exit; + } if (gpfsmpio_p2pcontig==1) { /* For some simple yet common(?) workloads, full-on two-phase I/O is overkill. We can establish sub-groups of processes and their aggregator, and then these sub-groups will carry out a simplified two-phase over that sub-group. * diff --git a/src/mpi/romio/adio/common/ad_close.c b/src/mpi/romio/adio/common/ad_close.c index 7aa5ce0..d51d75c 100644 --- a/src/mpi/romio/adio/common/ad_close.c +++ b/src/mpi/romio/adio/common/ad_close.c @@ -7,6 +7,7 @@ #include "adio.h" #include "adio_extern.h" +extern int gpfsmpio_aggmethod; #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -115,6 +116,10 @@ void ADIO_Close(ADIO_File fd, int *error_code) MPI_Info_free(&(fd->info)); if (fd->io_buf != NULL) ADIOI_Free(fd->io_buf); + /* If one-sided aggregation is chosen then free the window over the io_buf. + */ + if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) + MPI_Win_free(&fd->io_buf_window); /* memory for fd is freed in MPI_File_close */ } diff --git a/src/mpi/romio/adio/common/ad_open.c b/src/mpi/romio/adio/common/ad_open.c index 6558f76..c0d9331 100644 --- a/src/mpi/romio/adio/common/ad_open.c +++ b/src/mpi/romio/adio/common/ad_open.c @@ -126,6 +126,11 @@ MPI_File ADIO_Open(MPI_Comm orig_comm, * (e.g. Blue Gene) more efficent */ fd->io_buf = ADIOI_Malloc(fd->hints->cb_buffer_size); + /* If one-sided aggregation is chosen then create the window over the io_buf. + */ + if ((gpfsmpio_aggmethod == 1) || (gpfsmpio_aggmethod == 2)) { + MPI_Win_create(fd->io_buf,fd->hints->cb_buffer_size,1,MPI_INFO_NULL,fd->comm, &fd->io_buf_window); + } /* deferred open: * we can only do this optimization if 'fd->hints->deferred_open' is set * (which means the user hinted 'no_indep_rw' and collective buffering). http://git.mpich.org/mpich.git/commitdiff/147329674e7989e29c441efa73ba2c446… commit 147329674e7989e29c441efa73ba2c4462db5683 Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Sat Feb 28 00:45:32 2015 +0000 ROMIO Collective IO One-sided aggregation algorithm base code Optimized collective IO algorithm for GPFS to replace the existing two-phase algorithm with one utilizing one-sided MPI_Put and MPI_Get. Significant performance and memory optimization possible for certain workloads. Guarded by GPFSMPIO_AGGMETHOD environment variable, see ad_gpfs_tuning.c for details. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c index c993021..192f747 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c @@ -37,6 +37,7 @@ long bglocklessmpio_f_type; int gpfsmpio_bg_nagg_pset; int gpfsmpio_pthreadio; int gpfsmpio_p2pcontig; +int gpfsmpio_aggmethod; int gpfsmpio_balancecontig; int gpfsmpio_devnullio; int gpfsmpio_bridgeringagg; @@ -105,6 +106,22 @@ double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; * 3.) There are no gaps between the offsets. * 4.) No single rank has a data size which spans multiple file domains. * + * - GPFSMPIO_AGGMETHOD - Replaces the two-phase collective IO aggregation with a one- + * sided algorithm, significantly reducing communication and memory overhead. Fully + * supports all datasets and datatypes, the only caveat is that any holes in the data + * when writing to a pre-existing file are ignored -- there is no read-modify-write + * support to maintain the correctness of regions of pre-existing data so every byte + * must be explicitly written to maintain correctness. Users must beware of middle-ware + * libraries like PNETCDF which may count on read-modify-write functionality for certain + * features (like fill values). Possible values: + * - 0 - Normal two-phase collective IO is used. + * - 1 - A separate one-sided MPI_Put or MPI_Get is used for each contigous chunk of data + * for a compute to write to or read from the collective buffer on the aggregator. + * - 2 - An MPI derived datatype is created using all the contigous chunks and just one + * call to MPI_Put or MPI_Get is done with the derived datatype. On Blue Gene /Q + * optimal performance for this is achieved when paired with PAMID_TYPED_ONESIDED=1. + * - Default is 0 + * * - GPFSMPIO_BALANCECONTIG - Relevant only to BGQ. File domain blocks are assigned * to aggregators in a breadth-first fashion relative to the ions - additionally, * file domains on the aggregators sharing the same bridgeset and ion have contiguous @@ -165,6 +182,10 @@ void ad_gpfs_get_env_vars() { x = getenv( "GPFSMPIO_P2PCONTIG" ); if (x) gpfsmpio_p2pcontig = atoi(x); + gpfsmpio_aggmethod = 0; + x = getenv( "GPFSMPIO_AGGMETHOD" ); + if (x) gpfsmpio_aggmethod = atoi(x); + gpfsmpio_balancecontig = 0; x = getenv( "GPFSMPIO_BALANCECONTIG" ); if (x) gpfsmpio_balancecontig = atoi(x); diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h index b82c642..87aefce 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h @@ -66,6 +66,7 @@ extern int gpfsmpio_tuneblocking; extern long bglocklessmpio_f_type; extern int gpfsmpio_pthreadio; extern int gpfsmpio_p2pcontig; +extern int gpfsmpio_aggmethod; extern int gpfsmpio_balancecontig; extern int gpfsmpio_devnullio; extern int gpfsmpio_bridgeringagg; diff --git a/src/mpi/romio/adio/common/Makefile.mk b/src/mpi/romio/adio/common/Makefile.mk index c2c379c..80194ef 100644 --- a/src/mpi/romio/adio/common/Makefile.mk +++ b/src/mpi/romio/adio/common/Makefile.mk @@ -71,5 +71,6 @@ romio_other_sources += \ adio/common/hint_fns.c \ adio/common/ad_threaded_io.c \ adio/common/p2p_aggregation.c \ + adio/common/onesided_aggregation.c \ adio/common/utils.c diff --git a/src/mpi/romio/adio/common/ad_open.c b/src/mpi/romio/adio/common/ad_open.c index 7d69280..6558f76 100644 --- a/src/mpi/romio/adio/common/ad_open.c +++ b/src/mpi/romio/adio/common/ad_open.c @@ -10,7 +10,7 @@ #include "adio_cb_config_list.h" #include "mpio.h" - +extern int gpfsmpio_aggmethod; static int is_aggregator(int rank, ADIO_File fd); static int uses_generic_read(ADIO_File fd); static int uses_generic_write(ADIO_File fd); diff --git a/src/mpi/romio/adio/include/adio.h b/src/mpi/romio/adio/include/adio.h index 20ceb30..f1d4ac3 100644 --- a/src/mpi/romio/adio/include/adio.h +++ b/src/mpi/romio/adio/include/adio.h @@ -236,6 +236,7 @@ typedef struct ADIOI_FileD { MPI_Datatype *file_realm_types; /* file realm datatypes */ int my_cb_nodes_index; /* my index into cb_config_list. -1 if N/A */ char *io_buf; /* two-phase buffer allocated out of i/o path */ + MPI_Win io_buf_window; /* Window over the io_buf to support one-sided aggregation */ /* External32 */ int is_external32; /* bool: 0 means native view */ diff --git a/src/mpi/romio/adio/include/adioi.h b/src/mpi/romio/adio/include/adioi.h index b20ca82..6320a63 100644 --- a/src/mpi/romio/adio/include/adioi.h +++ b/src/mpi/romio/adio/include/adioi.h @@ -686,6 +686,28 @@ void ADIOI_P2PContigReadAggregation(ADIO_File fd, ADIO_Offset *fd_start, ADIO_Offset *fd_end); +void ADIOI_OneSidedWriteAggregation(ADIO_File fd, + ADIO_Offset *offset_list, + ADIO_Offset *len_list, + int contig_access_count, + const void *buf, + MPI_Datatype datatype, + int *error_code, + ADIO_Offset *st_offsets, + ADIO_Offset *end_offsets, + ADIO_Offset *fd_start, + ADIO_Offset* fd_end); +void ADIOI_OneSidedReadAggregation(ADIO_File fd, + ADIO_Offset *offset_list, + ADIO_Offset *len_list, + int contig_access_count, + const void *buf, + MPI_Datatype datatype, + int *error_code, + ADIO_Offset *st_offsets, + ADIO_Offset *end_offsets, + ADIO_Offset *fd_start, + ADIO_Offset* fd_end); ADIO_Offset ADIOI_GEN_SeekIndividual(ADIO_File fd, ADIO_Offset offset, int whence, int *error_code); void ADIOI_GEN_Resize(ADIO_File fd, ADIO_Offset size, int *error_code); http://git.mpich.org/mpich.git/commitdiff/0e4dcc43f3ef0be5fb035ceef310aeaf7… commit 0e4dcc43f3ef0be5fb035ceef310aeaf71e19df7 Author: Paul Coffman <pkcoff(a)vestalac1.ftd.alcf.anl.gov> Date: Sat Feb 28 00:35:43 2015 +0000 ROMIO One-sided Collective IO Aggregation base code Optimized collective IO aggregation algorithm added for GPFS which replaces the existing two-phase aggregation with one-sided MPI_Put and MPI_Get for writing and reading respectively. Significant performance and memory optimization possible for many workloads. Guarded by the GPFSMPIO_AGGMETHOD environment variable -- see ad_gpfs_tuning.c for details. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/common/onesided_aggregation.c b/src/mpi/romio/adio/common/onesided_aggregation.c new file mode 100644 index 0000000..695edde --- /dev/null +++ b/src/mpi/romio/adio/common/onesided_aggregation.c @@ -0,0 +1,1850 @@ +#include "adio.h" +#include "adio_extern.h" +#include "../ad_gpfs/ad_gpfs_tuning.h" + +#include <pthread.h> + +// #define onesidedtrace 1 + +/* Uncomment this to use fences for the one-sided communication. + */ +// #define ACTIVE_TARGET 1 + +/* This data structure holds the number of extents, the index into the flattened buffer and the remnant length + * beyond the flattened buffer index corresponding to the base buffer offset for non-contiguous source data + * for the range to be written coresponding to the round and target agg. + */ +typedef struct NonContigSourceBufOffset { + int dataTypeExtent; + int flatBufIndice; + ADIO_Offset remLen; +} NonContigSourceBufOffset; + +void ADIOI_OneSidedWriteAggregation(ADIO_File fd, + ADIO_Offset *offset_list, + ADIO_Offset *len_list, + int contig_access_count, + const void *buf, + MPI_Datatype datatype, + int *error_code, + ADIO_Offset *st_offsets, + ADIO_Offset *end_offsets, + ADIO_Offset *fd_start, + ADIO_Offset* fd_end) +{ + + *error_code = MPI_SUCCESS; /* initialize to success */ + +#ifdef ROMIO_GPFS + double startTimeBase,endTimeBase; + startTimeBase = MPI_Wtime(); +#endif + + int i,j; /* generic iterators */ + MPI_Status status; + pthread_t io_thread; + void *thread_ret; + ADIOI_IO_ThreadFuncData io_thread_args; + + int nprocs,myrank; + MPI_Comm_size(fd->comm, &nprocs); + MPI_Comm_rank(fd->comm, &myrank); + + /* This flag denotes whether the source datatype is contiguous, which is referenced throughout the algorithm + * and defines how the source buffer offsets and data chunks are determined. If the value is 1 (true - contiguous data) + * things are profoundly simpler in that the source buffer offset for a given target offset simply linearly increases + * by the chunk sizes being written. If the value is 0 (non-contiguous) then these values are based on calculations + * from the flattened source datatype. + */ + int bufTypeIsContig; + + MPI_Aint bufTypeExtent; + ADIOI_Flatlist_node *flatBuf=NULL; + ADIOI_Datatype_iscontig(datatype, &bufTypeIsContig); + + /* maxNumContigOperations keeps track of how many different chunks we will need to send + * for the purpose of pre-allocating the data structures to hold them. + */ + int maxNumContigOperations = contig_access_count; + + if (!bufTypeIsContig) { + /* Flatten the non-contiguous source datatype. + */ + ADIOI_Flatten_datatype(datatype); + flatBuf = ADIOI_Flatlist; + while (flatBuf->type != datatype) flatBuf = flatBuf->next; + MPI_Type_extent(datatype, &bufTypeExtent); + } + +#ifdef onesidedtrace + printf(" ADIOI_OneSidedWriteAggregation bufTypeIsContig is %d maxNumContigOperations is %d\n",bufTypeIsContig,maxNumContigOperations); +#endif + + ADIO_Offset myOffsetStart = st_offsets[myrank], myOffsetEnd = end_offsets[myrank], currentRoundOffsetStart = st_offsets[myrank]; + + ADIO_Offset lastFileOffset = 0, firstFileOffset = myOffsetEnd; + /* Get the total range being written. + */ + for (j=0;j<nprocs;j++) { + lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]); + firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]); + } + + int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */ + int iAmUsedAgg = 0; /* whether or not this rank is used as an aggregator. */ + + int naggs = fd->hints->cb_nodes; + int coll_bufsize = fd->hints->cb_buffer_size; +#ifdef ROMIO_GPFS + if (gpfsmpio_pthreadio == 1) { + /* split buffer in half for a kind of double buffering with the threads*/ + coll_bufsize = fd->hints->cb_buffer_size/2; + } +#endif + + /* This logic defines values that are used later to determine what offsets define the portion + * of the file domain the agg is writing this round. + */ + int greatestFileDomainAggRank = -1,smallestFileDomainAggRank = -1; + ADIO_Offset greatestFileDomainOffset = 0; + ADIO_Offset smallestFileDomainOffset = lastFileOffset; + for (j=0;j<naggs;j++) { + if (fd->hints->ranklist[j] == myrank) { + myAggRank = j; + if (fd_end[j] > greatestFileDomainOffset) { + greatestFileDomainOffset = fd_end[j]; + greatestFileDomainAggRank = j; + } + if (fd_start[j] < smallestFileDomainOffset) { + smallestFileDomainOffset = fd_start[j]; + smallestFileDomainAggRank = j; + } + if (fd_end[j] > fd_start[j]) { + iAmUsedAgg = 1; + } + } + } + +#ifdef onesidedtrace + printf("contig_access_count is %d lastFileOffset is %ld firstFileOffset is %ld\n",contig_access_count,lastFileOffset,firstFileOffset); + for (j=0;j<contig_access_count;j++) { + printf("offset_list[%d]: %ld , len_list[%d]: %ld\n",j,offset_list[j],j,len_list[j]); + } +#endif + + /* Determine how much data and to whom I need to send. For source proc + * targets, also determine the target file domain offsets locally to + * reduce communication overhead. + */ + int *targetAggsForMyData = (int *)ADIOI_Malloc(naggs * sizeof(int)); + ADIO_Offset *targetAggsForMyDataFDStart = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset)); + ADIO_Offset *targetAggsForMyDataFDEnd = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset)); + int numTargetAggs = 0; + + /* Compute number of rounds. + */ + ADIO_Offset numberOfRounds = (ADIO_Offset)((((ADIO_Offset)(end_offsets[nprocs-1]-st_offsets[0]))/((ADIO_Offset)((ADIO_Offset)coll_bufsize*(ADIO_Offset)naggs)))) + 1; + + /* This data structure holds the beginning offset and len list index for the range to be written + * coresponding to the round and target agg. Initialize to -1 to denote being unset. + */ + int **targetAggsForMyDataFirstOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *)); + for (i=0;i<numberOfRounds;i++) { + targetAggsForMyDataFirstOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int)); + for (j=0;j<naggs;j++) + targetAggsForMyDataFirstOffLenIndex[i][j] = -1; + } + + /* This data structure holds the ending offset and len list index for the range to be written + * coresponding to the round and target agg. + */ + int **targetAggsForMyDataLastOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *)); + for (i=0;i<numberOfRounds;i++) + targetAggsForMyDataLastOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int)); + + /* This data structure holds the base buffer offset for contiguous source data for the range to be written + * coresponding to the round and target agg. Initialize to -1 to denote being unset.yeah + */ + ADIO_Offset **baseSourceBufferOffset; + + if (bufTypeIsContig) { + baseSourceBufferOffset= (ADIO_Offset **)ADIOI_Malloc(numberOfRounds * sizeof(ADIO_Offset *)); + for (i=0;i<numberOfRounds;i++) { + baseSourceBufferOffset[i] = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset)); + for (j=0;j<naggs;j++) + baseSourceBufferOffset[i][j] = -1; + } + } + ADIO_Offset currentSourceBufferOffset = 0; + + /* This data structure holds the number of extents, the index into the flattened buffer and the remnant length + * beyond the flattened buffer indice corresponding to the base buffer offset for non-contiguous source data + * for the range to be written coresponding to the round and target agg. + */ + NonContigSourceBufOffset **baseNonContigSourceBufferOffset; + if (!bufTypeIsContig) { + baseNonContigSourceBufferOffset = (NonContigSourceBufOffset **) ADIOI_Malloc(numberOfRounds * sizeof(NonContigSourceBufOffset *)); + for (i=0;i<numberOfRounds;i++) { + baseNonContigSourceBufferOffset[i] = (NonContigSourceBufOffset *)ADIOI_Malloc(naggs * sizeof(NonContigSourceBufOffset)); + /* initialize flatBufIndice to -1 to indicate that it is unset. + */ + for (j=0;j<naggs;j++) + baseNonContigSourceBufferOffset[i][j].flatBufIndice = -1; + } + } + + int currentDataTypeExtent = 0; + int currentFlatBufIndice=0; + ADIO_Offset currentRemLen = 0; + +#ifdef onesidedtrace + printf("NumberOfRounds is %d\n",numberOfRounds); + for (i=0;i<naggs;i++) + printf("fd->hints->ranklist[%d]is %d fd_start is %ld fd_end is %ld\n",i,fd->hints->ranklist[i],fd_start[i],fd_end[i]); + for (j=0;j<contig_access_count;j++) + printf("offset_list[%d] is %ld len_list is %ld\n",j,offset_list[j],len_list[j]); +#endif + + int currentAggRankListIndex = 0; + + /* This denotes the coll_bufsize boundaries within the source buffer for writing for the same round. + */ + ADIO_Offset intraRoundCollBufsizeOffset = 0; + + /* This data structure tracks what target aggs need to be written to on what rounds. + */ + int *targetAggsForMyDataCurrentRoundIter = (int *)ADIOI_Malloc(naggs * sizeof(int)); + for (i=0;i<naggs;i++) + targetAggsForMyDataCurrentRoundIter[i] = 0; + + /* This is the first of the two main loops in this algorithm. The purpose of this loop is essentially to populate + * the data structures defined above for what source data blocks needs to go where (target agg) and when (round iter). + */ + int blockIter; + for (blockIter=0;blockIter<contig_access_count;blockIter++) { + + /* Determine the starting source buffer offset for this block - for iter 0 skip it since that value is 0. + */ + if (blockIter>0) { + if (bufTypeIsContig) + currentSourceBufferOffset += len_list[blockIter-1]; + else { + /* Non-contiguous source datatype, count up the extents and indices to this point + * in the blocks for use in computing the source buffer offset. + */ + ADIO_Offset sourceBlockTotal = 0-currentRemLen; + while (sourceBlockTotal < len_list[blockIter-1]) { + maxNumContigOperations++; + sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + currentFlatBufIndice++; + if (currentFlatBufIndice == flatBuf->count) { + currentFlatBufIndice = 0; + currentDataTypeExtent++; + } + } + if (sourceBlockTotal > len_list[blockIter-1]) { + currentFlatBufIndice--; + if (currentFlatBufIndice < 0 ) { + currentDataTypeExtent--; + currentFlatBufIndice = flatBuf->count; + } + } + currentRemLen = len_list[blockIter-1] - sourceBlockTotal; +#ifdef onesidedtrace + printf("contig_access_count iter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); +#endif + } // !bufTypeIsContig + } // blockIter > 0 + + /* For the first iteration we need to include these maxNumContigOperations + * for non-contig case even though we did not need to compute the starting offset. + */ + if ((blockIter == 0) && (!bufTypeIsContig)) { + ADIO_Offset sourceBlockTotal = 0; + int tmpCurrentFlatBufIndice = currentFlatBufIndice; + while (sourceBlockTotal < len_list[0]) { + maxNumContigOperations++; + sourceBlockTotal += flatBuf->blocklens[tmpCurrentFlatBufIndice]; + tmpCurrentFlatBufIndice++; + if (tmpCurrentFlatBufIndice == flatBuf->count) { + tmpCurrentFlatBufIndice = 0; + } + } + } + + ADIO_Offset blockStart = offset_list[blockIter], blockLen = len_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1; + + /* Find the starting target agg for this block - normally it will be the current agg so guard the expensive + * while loop with a cheap if-check which for large numbers of small blocks will usually be false. + */ + if (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex]))) { + while (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex]))) + currentAggRankListIndex++; + }; + +#ifdef onesidedtrace + printf("currentAggRankListIndex is %d blockStart %ld blockEnd %ld fd_start[currentAggRankListIndex] %ld fd_end[currentAggRankListIndex] %ld\n",currentAggRankListIndex,blockStart,blockEnd,fd_start[currentAggRankListIndex],fd_end[currentAggRankListIndex]); +#endif + + /* Determine if this is a new target agg. + */ + if (blockIter>0) { + if ((offset_list[blockIter-1]+len_list[blockIter-1]-(ADIO_Offset)1) < fd_start[currentAggRankListIndex]) + numTargetAggs++; + } + + /* Determine which round to start writing. + */ + if ((blockStart - fd_start[currentAggRankListIndex]) >= coll_bufsize) { + ADIO_Offset currentRoundBlockStart = fd_start[currentAggRankListIndex]; + int startingRound = 0; + while (blockStart > (currentRoundBlockStart + coll_bufsize - (ADIO_Offset)1)) { + currentRoundBlockStart+=coll_bufsize; + startingRound++; + } + targetAggsForMyDataCurrentRoundIter[numTargetAggs] = startingRound; + } + + /* Initialize the data structures if this is the first offset in the round/target agg. + */ + if (targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] == -1) { + targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex]; + targetAggsForMyDataFDStart[numTargetAggs] = fd_start[currentAggRankListIndex]; + targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex]; + targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; + if (bufTypeIsContig) + baseSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = currentSourceBufferOffset; + else { + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + } + + intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + ((targetAggsForMyDataCurrentRoundIter[numTargetAggs]+1) * coll_bufsize); + +#ifdef onesidedtrace + printf("Initial settings numTargetAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld currentSourceBufferOffset set to %ld intraRoundCollBufsizeOffset set to %ld\n",numTargetAggs,blockIter,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter],currentSourceBufferOffset,intraRoundCollBufsizeOffset); +#endif + } + + /* Replace the last offset block iter with this one. + */ + targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; + + /* If this blocks extends into the next file domain handle this situation. + */ + if (blockEnd > fd_end[currentAggRankListIndex]) { +#ifdef onesidedtrace + printf("large block, blockEnd %ld >= fd_end[currentAggRankListIndex] %ld\n",blockEnd,fd_end[currentAggRankListIndex]); +#endif + while (blockEnd >= fd_end[currentAggRankListIndex]) { + ADIO_Offset thisAggBlockEnd = fd_end[currentAggRankListIndex]; + if (thisAggBlockEnd >= intraRoundCollBufsizeOffset) { + while (thisAggBlockEnd >= intraRoundCollBufsizeOffset) { + targetAggsForMyDataCurrentRoundIter[numTargetAggs]++; + intraRoundCollBufsizeOffset += coll_bufsize; + targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; + if (bufTypeIsContig) + baseSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = currentSourceBufferOffset; + else { + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + } + + targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; +#ifdef onesidedtrace + printf("targetAggsForMyDataCurrentRoundI%d] is now %d intraRoundCollBufsizeOffset is now %ld\n",numTargetAggs,targetAggsForMyDataCurrentRoundIter[numTargetAggs],intraRoundCollBufsizeOffset); +#endif + } // while (thisAggBlockEnd >= intraRoundCollBufsizeOffset) + } // if (thisAggBlockEnd >= intraRoundCollBufsizeOffset) + + currentAggRankListIndex++; + + /* Skip over unused aggs. + */ + if (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) { + while (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) + currentAggRankListIndex++; + + } // (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) + + /* Start new target agg. + */ + if (blockEnd >= fd_start[currentAggRankListIndex]) { + numTargetAggs++; + targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex]; + targetAggsForMyDataFDStart[numTargetAggs] = fd_start[currentAggRankListIndex]; + targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex]; + targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; + if (bufTypeIsContig) + baseSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = currentSourceBufferOffset; + else { + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + } +#ifdef onesidedtrace + printf("large block init settings numTargetAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld\n",numTargetAggs,i,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter]); +#endif + intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + coll_bufsize; + targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; + } // if (blockEnd >= fd_start[currentAggRankListIndex]) + } // while (blockEnd >= fd_end[currentAggRankListIndex]) + } // if (blockEnd > fd_end[currentAggRankListIndex]) + + /* Else if we are still in the same file domain / target agg but have gone past the coll_bufsize and need + * to advance to the next round handle this situation. + */ + else if (blockEnd >= intraRoundCollBufsizeOffset) { + ADIO_Offset currentBlockEnd = blockEnd; + while (currentBlockEnd >= intraRoundCollBufsizeOffset) { + targetAggsForMyDataCurrentRoundIter[numTargetAggs]++; + intraRoundCollBufsizeOffset += coll_bufsize; + targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; + if (bufTypeIsContig) + baseSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = currentSourceBufferOffset; + else { + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs].remLen = currentRemLen; + } + targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter; +#ifdef onesidedtrace + printf("smaller than fd currentBlockEnd is now %ld intraRoundCollBufsizeOffset is now %ld targetAggsForMyDataCurrentRoundIter[%d] is now %d\n",currentBlockEnd, intraRoundCollBufsizeOffset, numTargetAggs,targetAggsForMyDataCurrentRoundIter[numTargetAggs]); +#endif + } // while (currentBlockEnd >= intraRoundCollBufsizeOffset) + } // else if (blockEnd >= intraRoundCollBufsizeOffset) + + /* Need to advance numTargetAggs if this is the last target offset to + * include this one. + */ + if (blockIter == (contig_access_count-1)) + numTargetAggs++; + } + +#ifdef onesidedtrace + printf("numTargetAggs is %d\n",numTargetAggs); + for (i=0;i<numTargetAggs;i++) { + for (j=0;j<=targetAggsForMyDataCurrentRoundIter[i];j++) + printf("targetAggsForMyData[%d] is %d targetAggsForMyDataFDStart[%d] is %ld targetAggsForMyDataFDEnd is %ld targetAggsForMyDataFirstOffLenIndex is %d with value %ld targetAggsForMyDataLastOffLenIndex is %d with value %ld\n",i,targetAggsForMyData[i],i,targetAggsForMyDataFDStart[i],targetAggsForMyDataFDEnd[i],targetAggsForMyDataFirstOffLenIndex[j][i],offset_list[targetAggsForMyDataFirstOffLenIndex[j][i]],targetAggsForMyDataLastOffLenIndex[j][i],offset_list[targetAggsForMyDataLastOffLenIndex[j][i]]); + + } +#endif + + ADIOI_Free(targetAggsForMyDataCurrentRoundIter); + + int currentWriteBuf = 0; + int useIOBuffer = 0; +#ifdef ROMIO_GPFS + if (gpfsmpio_pthreadio && (numberOfRounds>1)) { + useIOBuffer = 1; + io_thread = pthread_self(); + } +#endif + + /* use the write buffer allocated in the file_open */ + char *write_buf0 = fd->io_buf; + char *write_buf1 = fd->io_buf + coll_bufsize; + + /* start off pointing to the first buffer. If we use the 2nd buffer (threaded + * case) we'll swap later */ + char *write_buf = write_buf0; + MPI_Win write_buf_window = fd->io_buf_window; + +#ifdef ACTIVE_TARGET + MPI_Win_fence(0, write_buf_window); +#endif + + ADIO_Offset currentRoundFDStart = 0; + ADIO_Offset currentRoundFDEnd = 0; + + if (iAmUsedAgg) { + currentRoundFDStart = fd_start[myAggRank]; + if (myAggRank == smallestFileDomainAggRank) { + if (currentRoundFDStart < firstFileOffset) + currentRoundFDStart = firstFileOffset; + } + } + +#ifdef ROMIO_GPFS + endTimeBase = MPI_Wtime(); + gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SETUP] += (endTimeBase-startTimeBase); + startTimeBase = MPI_Wtime(); +#endif + + ADIO_Offset currentBaseSourceBufferOffset = 0; + + /* This is the second main loop of the algorithm, actually nested loop of target aggs within rounds. There are 2 flavors of this. + * For gpfsmpio_aggmethod of 1 each nested iteration for the target agg does an mpi_put on a contiguous chunk using a primative datatype + * determined using the data structures from the first main loop. For gpfsmpio_aggmethod of 2 each nested iteration for the target agg + * builds up data to use in created a derived data type for 1 mpi_put that is done for the target agg for each round. + */ + int roundIter; + for (roundIter=0;roundIter<numberOfRounds;roundIter++) { + + int aggIter; + for (aggIter=0;aggIter<numTargetAggs;aggIter++) { + + /* If we have data for the round/agg process it. + */ + if ((bufTypeIsContig && (baseSourceBufferOffset[roundIter][aggIter] != -1)) || (!bufTypeIsContig && (baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice != -1))) { + + ADIO_Offset currentRoundFDStartForMyTargetAgg = (ADIO_Offset)((ADIO_Offset)targetAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)roundIter*(ADIO_Offset)coll_bufsize)); + ADIO_Offset currentRoundFDEndForMyTargetAgg = (ADIO_Offset)((ADIO_Offset)targetAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)(roundIter+1)*(ADIO_Offset)coll_bufsize) - (ADIO_Offset)1); + + int targetAggContigAccessCount = 0; + + /* These data structures are used for the derived datatype mpi_put + * in the gpfsmpio_aggmethod of 2 case. + */ + int *targetAggBlockLengths; + MPI_Aint *targetAggDisplacements, *sourceBufferDisplacements; + MPI_Datatype *targetAggDataTypes; + + int allocatedDerivedTypeArrays = 0; +#ifdef onesidedtrace + printf("roundIter %d processing targetAggsForMyData %d \n",roundIter,targetAggsForMyData[aggIter]); +#endif + + ADIO_Offset sourceBufferOffset = 0; + + /* Process the range of offsets for this target agg. + */ + int offsetIter; + int startingOffLenIndex = targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter], endingOffLenIndex = targetAggsForMyDataLastOffLenIndex[roundIter][aggIter]; + for (offsetIter=startingOffLenIndex;offsetIter<=endingOffLenIndex;offsetIter++) { + if (currentRoundFDEndForMyTargetAgg > targetAggsForMyDataFDEnd[aggIter]) + currentRoundFDEndForMyTargetAgg = targetAggsForMyDataFDEnd[aggIter]; + + ADIO_Offset offsetStart = offset_list[offsetIter], offsetEnd = (offset_list[offsetIter]+len_list[offsetIter]-(ADIO_Offset)1); + + /* Get the base source buffer offset for this iterataion of the target agg in the round. + * For the first one just get the predetermined base value, for subsequent ones keep track + * of the value for each iteration and increment it. + */ + if (offsetIter == startingOffLenIndex) { + if (bufTypeIsContig) + currentBaseSourceBufferOffset = baseSourceBufferOffset[roundIter][aggIter]; + else { + currentRemLen = baseNonContigSourceBufferOffset[roundIter][aggIter].remLen; + currentDataTypeExtent = baseNonContigSourceBufferOffset[roundIter][aggIter].dataTypeExtent; + currentFlatBufIndice = baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice; + } + } + else { + if (bufTypeIsContig) + currentBaseSourceBufferOffset += len_list[offsetIter-1]; + else { + + /* For non-contiguous source datatype count up the extents and indices to this point. + */ + ADIO_Offset sourceBlockTotal = 0-currentRemLen; + while (sourceBlockTotal < len_list[offsetIter-1]) { + sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + currentFlatBufIndice++; + if (currentFlatBufIndice == flatBuf->count) { + currentFlatBufIndice = 0; + currentDataTypeExtent++; + } + } // while + if (sourceBlockTotal > len_list[offsetIter-1]) + currentFlatBufIndice--; + currentRemLen = len_list[offsetIter-1] - sourceBlockTotal; +#ifdef onesidedtrace + printf("contig_access_count target agg %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",aggIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); +#endif + } + } + + /* For source buffer contiguous here is the offset we will use for this iteration. + */ + if (bufTypeIsContig) + sourceBufferOffset = currentBaseSourceBufferOffset; + +#ifdef onesidedtrace + printf("roundIter %d target iter %d targetAggsForMyData is %d offset_list[%d] is %ld len_list[%d] is %ld targetAggsForMyDataFDStart is %ld targetAggsForMyDataFDEnd is %ld currentRoundFDStartForMyTargetAgg is %ld currentRoundFDEndForMyTargetAgg is %ld targetAggsForMyDataFirstOffLenIndex is %ld baseSourceBufferOffset is %ld\n", + roundIter,aggIter,targetAggsForMyData[aggIter],offsetIter,offset_list[offsetIter],offsetIter,len_list[offsetIter], + targetAggsForMyDataFDStart[aggIter],targetAggsForMyDataFDEnd[aggIter], + currentRoundFDStartForMyTargetAgg,currentRoundFDEndForMyTargetAgg, targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter], baseSourceBufferOffset[roundIter][aggIter]); +#endif + + /* Determine the amount of data and exact source buffer offsets to use. + */ + int bufferAmountToSend = 0; + + if ((offsetStart >= currentRoundFDStartForMyTargetAgg) && (offsetStart <= currentRoundFDEndForMyTargetAgg)) { + if (offsetEnd > currentRoundFDEndForMyTargetAgg) + bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - offsetStart) +1; + else + bufferAmountToSend = (offsetEnd - offsetStart) +1; + } + else if ((offsetEnd >= currentRoundFDStartForMyTargetAgg) && (offsetEnd <= currentRoundFDEndForMyTargetAgg)) { + if (offsetEnd > currentRoundFDEndForMyTargetAgg) + bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - currentRoundFDStartForMyTargetAgg) +1; + else + bufferAmountToSend = (offsetEnd - currentRoundFDStartForMyTargetAgg) +1; + if (offsetStart < currentRoundFDStartForMyTargetAgg) { + if (bufTypeIsContig) + sourceBufferOffset += (currentRoundFDStartForMyTargetAgg-offsetStart); + offsetStart = currentRoundFDStartForMyTargetAgg; + } + } + else if ((offsetStart <= currentRoundFDStartForMyTargetAgg) && (offsetEnd >= currentRoundFDEndForMyTargetAgg)) { + bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - currentRoundFDStartForMyTargetAgg) +1; + if (bufTypeIsContig) + sourceBufferOffset += (currentRoundFDStartForMyTargetAgg-offsetStart); + offsetStart = currentRoundFDStartForMyTargetAgg; + } + + if (bufferAmountToSend > 0) { /* we have data to send this round */ + if (gpfsmpio_aggmethod == 2) { + /* Only allocate these arrays if we are using method 2 and only do it once for this round/target agg. + */ + if (!allocatedDerivedTypeArrays) { + targetAggBlockLengths = (int *)ADIOI_Malloc(maxNumContigOperations * sizeof(int)); + targetAggDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint)); + sourceBufferDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint)); + targetAggDataTypes = (MPI_Datatype *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Datatype)); + allocatedDerivedTypeArrays = 1; + } + } + /* For the non-contiguous source datatype we need to determine the number,size and offsets of the chunks + * from the source buffer to be sent to this contiguous chunk defined by the round/agg/iter in the target. + */ + int numNonContigSourceChunks = 0; + ADIO_Offset *nonContigSourceOffsets; + int *nonContigSourceLens; + ADIO_Offset baseDatatypeInstanceOffset = 0; + + if (!bufTypeIsContig) { + + currentSourceBufferOffset = (ADIO_Offset)((ADIO_Offset)currentDataTypeExtent * (ADIO_Offset)bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentRemLen; +#ifdef onesidedtrace + printf("!bufTypeIsContig currentSourceBufferOffset set to %ld for roundIter %d target %d currentDataTypeExtent %d flatBuf->indices[currentFlatBufIndice] %ld currentRemLen %ld currentFlatBufIndice %d\n",currentSourceBufferOffset,roundIter,aggIter,currentDataTypeExtent,flatBuf->indices[currentFlatBufIndice],currentRemLen,currentFlatBufIndice); +#endif + + /* Count the chunks first to see how much to malloc from the ending point from above code. + */ + int sendBytesCounted = 0; + int tmpFlatBufIndice = currentFlatBufIndice; + int maxNumNonContigSourceChunks = 2; // over-initialize for potential remnants on both ends + + while (sendBytesCounted < bufferAmountToSend) { + maxNumNonContigSourceChunks++; + if (tmpFlatBufIndice == flatBuf->count) { + tmpFlatBufIndice = 0; + } + sendBytesCounted += flatBuf->blocklens[tmpFlatBufIndice]; + tmpFlatBufIndice++; + } + + nonContigSourceOffsets = (ADIO_Offset *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(ADIO_Offset)); + nonContigSourceLens = (int *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(int)); + + /* now populate the nonContigSourceOffsets and nonContigSourceLens arrays for use in the one-sided operations. + */ + int ncArrayIndex = 0; + int remainingBytesToLoadedIntoNCArrays = bufferAmountToSend; + ADIO_Offset indexIntoCurrentIndice = 0; + if (currentRemLen > 0) + indexIntoCurrentIndice = flatBuf->blocklens[currentFlatBufIndice] - currentRemLen; + + int datatypeInstances = currentDataTypeExtent; + tmpFlatBufIndice = currentFlatBufIndice; + while (remainingBytesToLoadedIntoNCArrays > 0) { + nonContigSourceOffsets[ncArrayIndex] = currentSourceBufferOffset; + + if ((flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice) > remainingBytesToLoadedIntoNCArrays) { + nonContigSourceLens[ncArrayIndex] = remainingBytesToLoadedIntoNCArrays; + remainingBytesToLoadedIntoNCArrays = 0; + } + else { + nonContigSourceLens[ncArrayIndex] = (int)(flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); + remainingBytesToLoadedIntoNCArrays -= (flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); + } + indexIntoCurrentIndice = 0; // only worry about beginning remnant for first iter + + tmpFlatBufIndice++; + if (tmpFlatBufIndice == flatBuf->count) { + tmpFlatBufIndice = 0; + datatypeInstances++; + baseDatatypeInstanceOffset = datatypeInstances * bufTypeExtent; + } + currentSourceBufferOffset = baseDatatypeInstanceOffset + flatBuf->indices[tmpFlatBufIndice]; + ncArrayIndex++; + numNonContigSourceChunks++; + } // while +#ifdef onesidedtrace + printf("CurrentSourceBufferOffset finally set to %ld\n",currentSourceBufferOffset); +#endif + } // !bufTypeIsContig + + + /* Determine the offset into the target window. + */ + MPI_Aint targetDisplacementToUseThisRound = (MPI_Aint) ((ADIO_Offset)offsetStart - currentRoundFDStartForMyTargetAgg); + + /* If using the thread writer select the appropriate side of the split window. + */ + if (useIOBuffer && (write_buf == write_buf1)) { + targetDisplacementToUseThisRound += coll_bufsize; + } + + /* For gpfsmpio_aggmethod of 1 do the mpi_put using the primitive MPI_BYTE type on each contiguous chunk of source data. + */ + if (gpfsmpio_aggmethod == 1) { +#ifndef ACTIVE_TARGET + MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, write_buf_window); +#endif + if (bufTypeIsContig) { + MPI_Put(&((char*)buf)[sourceBufferOffset],bufferAmountToSend, MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, bufferAmountToSend,MPI_BYTE,write_buf_window); + } + else { + for (i=0;i<numNonContigSourceChunks;i++) { + MPI_Put(&((char*)buf)[nonContigSourceOffsets[i]],nonContigSourceLens[i], MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, nonContigSourceLens[i],MPI_BYTE,write_buf_window); +#ifdef onesidedtrace + printf("mpi_put[%d] nonContigSourceOffsets is %d of nonContigSourceLens %d to target disp %d\n",i,nonContigSourceOffsets[i],nonContigSourceLens[i],targetDisplacementToUseThisRound); +#endif + targetDisplacementToUseThisRound += nonContigSourceLens[i]; + } + } +#ifndef ACTIVE_TARGET + MPI_Win_unlock(targetAggsForMyData[aggIter], write_buf_window); +#endif + } + + /* For gpfsmpio_aggmethod of 2 populate the data structures for this round/agg for this offset iter + * to be used subsequently when building the derived type for 1 mpi_put for all the data for this + * round/agg. + */ + else if (gpfsmpio_aggmethod == 2) { + + if (bufTypeIsContig) { + targetAggBlockLengths[targetAggContigAccessCount]= bufferAmountToSend; + targetAggDataTypes[targetAggContigAccessCount] = MPI_BYTE; + targetAggDisplacements[targetAggContigAccessCount] = targetDisplacementToUseThisRound; + sourceBufferDisplacements[targetAggContigAccessCount] = (MPI_Aint)sourceBufferOffset; + targetAggContigAccessCount++; + } + else { + for (i=0;i<numNonContigSourceChunks;i++) { + targetAggBlockLengths[targetAggContigAccessCount]= nonContigSourceLens[i]; + targetAggDataTypes[targetAggContigAccessCount] = MPI_BYTE; + targetAggDisplacements[targetAggContigAccessCount] = targetDisplacementToUseThisRound; + sourceBufferDisplacements[targetAggContigAccessCount] = (MPI_Aint)nonContigSourceOffsets[i]; +#ifdef onesidedtrace + printf("mpi_put building arrays for iter %d - sourceBufferDisplacements is %d of nonContigSourceLens %d to target disp %d targetAggContigAccessCount is %d targetAggBlockLengths[targetAggContigAccessCount] is %d\n",i,sourceBufferDisplacements[targetAggContigAccessCount],nonContigSourceLens[i],targetAggDisplacements[targetAggContigAccessCount],targetAggContigAccessCount, targetAggBlockLengths[targetAggContigAccessCount]); +#endif + targetAggContigAccessCount++; + targetDisplacementToUseThisRound += nonContigSourceLens[i]; + } + } + } +#ifdef onesidedtrace + printf("roundIter %d bufferAmountToSend is %d sourceBufferOffset is %d offsetStart is %ld currentRoundFDStartForMyTargetAgg is %ld targetDisplacementToUseThisRound is %ld targetAggsForMyDataFDStart[aggIter] is %ld\n",roundIter, bufferAmountToSend,sourceBufferOffset, offsetStart,currentRoundFDStartForMyTargetAgg,targetDisplacementToUseThisRound,targetAggsForMyDataFDStart[aggIter]); +#endif + + if (!bufTypeIsContig) { + ADIOI_Free(nonContigSourceOffsets); + ADIOI_Free(nonContigSourceLens); + } + } // bufferAmountToSend > 0 + } // contig list + + /* For gpfsmpio_aggmethod of 2 now build the derived type using the data from this round/agg and do 1 single mpi_put. + */ + if (gpfsmpio_aggmethod == 2) { + MPI_Datatype sourceBufferDerivedDataType, targetBufferDerivedDataType; + MPI_Type_create_struct(targetAggContigAccessCount, targetAggBlockLengths, sourceBufferDisplacements, targetAggDataTypes, &sourceBufferDerivedDataType); + MPI_Type_commit(&sourceBufferDerivedDataType); + MPI_Type_create_struct(targetAggContigAccessCount, targetAggBlockLengths, targetAggDisplacements, targetAggDataTypes, &targetBufferDerivedDataType); + MPI_Type_commit(&targetBufferDerivedDataType); + +#ifdef onesidedtrace + printf("mpi_put of derived type to agg %d targetAggContigAccessCount is %d\n",targetAggsForMyData[aggIter],targetAggContigAccessCount); +#endif +#ifndef ACTIVE_TARGET + MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, write_buf_window); +#endif + + MPI_Put(((char*)buf),1, sourceBufferDerivedDataType,targetAggsForMyData[aggIter],0, 1,targetBufferDerivedDataType,write_buf_window); + +#ifndef ACTIVE_TARGET + MPI_Win_unlock(targetAggsForMyData[aggIter], write_buf_window); +#endif + + if (allocatedDerivedTypeArrays) { + ADIOI_Free(targetAggBlockLengths); + ADIOI_Free(targetAggDisplacements); + ADIOI_Free(targetAggDataTypes); + ADIOI_Free(sourceBufferDisplacements); + } + MPI_Type_free(&sourceBufferDerivedDataType); + MPI_Type_free(&targetBufferDerivedDataType); + } + } // baseoffset != -1 + } // target aggs + + /* the source procs send the requested data to the aggs */ + +#ifdef ACTIVE_TARGET + MPI_Win_fence(0, write_buf_window); +#else + MPI_Barrier(fd->comm); +#endif + + if (iAmUsedAgg) { + /* Determine what offsets define the portion of the file domain the agg is writing this round. + */ + if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) { + if (myAggRank == greatestFileDomainAggRank) { + if (fd_end[myAggRank] > lastFileOffset) + currentRoundFDEnd = lastFileOffset; + else + currentRoundFDEnd = fd_end[myAggRank]; + } + else + currentRoundFDEnd = fd_end[myAggRank]; + } + else + currentRoundFDEnd = currentRoundFDStart + coll_bufsize - 1; + +#ifdef onesidedtrace + printf("currentRoundFDStart is %ld currentRoundFDEnd is %ld within file domeain %ld to %ld\n",currentRoundFDStart,currentRoundFDEnd,fd_start[myAggRank],fd_end[myAggRank]); +#endif + + if (!useIOBuffer) { + ADIO_WriteContig(fd, write_buf, (int)(currentRoundFDEnd - currentRoundFDStart)+1, + MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code); + int numBytesWritten= (int)(currentRoundFDEnd - currentRoundFDStart)+1; + +/* For now this algorithm cannot handle holes in the source data and does not do any data sieving. + * One possible approach would be to initialize the write buffer with some value and then check to + * see if the mpi_put operations changed the values, if they were unchanged then retry with some other + * default initialized value and if that was still unchanged then you would know where a hole was. + * Here is some initial sample code for that: + * if (roundIter<(numberOfRounds-1)) { + * for (i=0;i<numBytesWritten;i++) + * write_buf[i] = '\0'; + * } +*/ + } else { /* use the thread writer */ + + if(!pthread_equal(io_thread, pthread_self())) { + pthread_join(io_thread, &thread_ret); + *error_code = *(int *)thread_ret; + if (*error_code != MPI_SUCCESS) return; + io_thread = pthread_self(); + + } + io_thread_args.fd = fd; + /* do a little pointer shuffling: background I/O works from one + * buffer while two-phase machinery fills up another */ + + if (currentWriteBuf == 0) { + io_thread_args.buf = write_buf0; + currentWriteBuf = 1; + write_buf = write_buf1; + } + else { + io_thread_args.buf = write_buf1; + currentWriteBuf = 0; + write_buf = write_buf0; + } + io_thread_args.io_kind = ADIOI_WRITE; + io_thread_args.size = (currentRoundFDEnd-currentRoundFDStart) + 1; + io_thread_args.offset = currentRoundFDStart; + io_thread_args.status = status; + io_thread_args.error_code = *error_code; + + if ( (pthread_create(&io_thread, NULL, + ADIOI_IO_Thread_Func, &(io_thread_args))) != 0) + io_thread = pthread_self(); + + } // useIOBuffer + + } // iAmUsedAgg + + if (!iAmUsedAgg && useIOBuffer) { + if (currentWriteBuf == 0) { + currentWriteBuf = 1; + write_buf = write_buf1; + } + else { + currentWriteBuf = 0; + write_buf = write_buf0; + } + } + + if (iAmUsedAgg) + currentRoundFDStart += coll_bufsize; + + if (roundIter<(numberOfRounds-1)) + MPI_Barrier(fd->comm); + + } /* for-loop roundIter */ + +#ifdef ROMIO_GPFS + endTimeBase = MPI_Wtime(); + gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase); +#endif + + if (useIOBuffer) { /* thread writer cleanup */ + + if ( !pthread_equal(io_thread, pthread_self()) ) { + pthread_join(io_thread, &thread_ret); + *error_code = *(int *)thread_ret; + } + + } + + ADIOI_Free(targetAggsForMyData); + ADIOI_Free(targetAggsForMyDataFDStart); + ADIOI_Free(targetAggsForMyDataFDEnd); + + for (i=0;i<numberOfRounds;i++) { + ADIOI_Free(targetAggsForMyDataFirstOffLenIndex[i]); + ADIOI_Free(targetAggsForMyDataLastOffLenIndex[i]); + if (bufTypeIsContig) + ADIOI_Free(baseSourceBufferOffset[i]); + else + ADIOI_Free(baseNonContigSourceBufferOffset[i]); + } + ADIOI_Free(targetAggsForMyDataFirstOffLenIndex); + ADIOI_Free(targetAggsForMyDataLastOffLenIndex); + if (bufTypeIsContig) + ADIOI_Free(baseSourceBufferOffset); + else + ADIOI_Free(baseNonContigSourceBufferOffset); + + return; +} + + +void ADIOI_OneSidedReadAggregation(ADIO_File fd, + ADIO_Offset *offset_list, + ADIO_Offset *len_list, + int contig_access_count, + const void *buf, + MPI_Datatype datatype, + int *error_code, + ADIO_Offset *st_offsets, + ADIO_Offset *end_offsets, + ADIO_Offset *fd_start, + ADIO_Offset* fd_end) +{ + + *error_code = MPI_SUCCESS; /* initialize to success */ + +#ifdef ROMIO_GPFS + double startTimeBase,endTimeBase; + startTimeBase = MPI_Wtime(); +#endif + + MPI_Status status; + pthread_t io_thread; + void *thread_ret; + ADIOI_IO_ThreadFuncData io_thread_args; + int i,j; /* generic iterators */ + + int nprocs,myrank; + MPI_Comm_size(fd->comm, &nprocs); + MPI_Comm_rank(fd->comm, &myrank); + + /* This flag denotes whether the source datatype is contiguus, which is referenced throughout the algorithm + * and defines how the source buffer offsets and data chunks are determined. If the value is 1 (true - contiguous data) + * things are profoundly simpler in that the source buffer offset for a given source offset simply linearly increases + * by the chunk sizes being read. If the value is 0 (non-contiguous) then these values are based on calculations + * from the flattened source datatype. + */ + int bufTypeIsContig; + + MPI_Aint bufTypeExtent; + ADIOI_Flatlist_node *flatBuf=NULL; + ADIOI_Datatype_iscontig(datatype, &bufTypeIsContig); + + /* maxNumContigOperations keeps track of how many different chunks we will need to recv + * for the purpose of pre-allocating the data structures to hold them. + */ + int maxNumContigOperations = contig_access_count; + + if (!bufTypeIsContig) { + /* Flatten the non-contiguous source datatype. + */ + ADIOI_Flatten_datatype(datatype); + flatBuf = ADIOI_Flatlist; + while (flatBuf->type != datatype) flatBuf = flatBuf->next; + MPI_Type_extent(datatype, &bufTypeExtent); + } +#ifdef onesidedtrace + printf("ADIOI_OneSidedReadAggregation bufTypeIsContig is %d maxNumContigOperations is %d\n",bufTypeIsContig,maxNumContigOperations); +#endif + + ADIO_Offset myOffsetStart = st_offsets[myrank], myOffsetEnd = end_offsets[myrank], currentRoundOffsetStart = st_offsets[myrank]; + + ADIO_Offset lastFileOffset = 0, firstFileOffset = myOffsetEnd; + + /* Get the total range being read. + */ + for (j=0;j<nprocs;j++) { + lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]); + firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]); + } + + int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */ + int iAmUsedAgg = 0; /* whether or not this rank is used as an aggregator. */ + + int naggs = fd->hints->cb_nodes; + int coll_bufsize = fd->hints->cb_buffer_size; +#ifdef ROMIO_GPFS + if (gpfsmpio_pthreadio == 1) { + /* split buffer in half for a kind of double buffering with the threads*/ + coll_bufsize = fd->hints->cb_buffer_size/2; + } +#endif + + /* This logic defines values that are used later to determine what offsets define the portion + * of the file domain the agg is reading this round. + */ + int greatestFileDomainAggRank = -1,smallestFileDomainAggRank = -1; + ADIO_Offset greatestFileDomainOffset = 0; + ADIO_Offset smallestFileDomainOffset = lastFileOffset; + for (j=0;j<naggs;j++) { + if (fd->hints->ranklist[j] == myrank) { + myAggRank = j; + if (fd_end[j] > greatestFileDomainOffset) { + greatestFileDomainOffset = fd_end[j]; + greatestFileDomainAggRank = j; + } + if (fd_start[j] < smallestFileDomainOffset) { + smallestFileDomainOffset = fd_start[j]; + smallestFileDomainAggRank = j; + } + if (fd_end[j] > fd_start[j]) { + iAmUsedAgg = 1; + } + } + } + +#ifdef onesidedtrace + printf("contig_access_count is %d lastFileOffset is %ld firstFileOffset is %ld\n",contig_access_count,lastFileOffset,firstFileOffset); + for (j=0;j<contig_access_count;j++) { + printf("offset_list[%d]: %ld , len_list[%d]: %ld\n",j,offset_list[j],j,len_list[j]); + } +#endif + + + /* for my offset range determine how much data and from whom I need to get + * it. For source ag sources, also determine the source file domain + * offsets locally to reduce communication overhead */ + int *sourceAggsForMyData = (int *)ADIOI_Malloc(naggs * sizeof(int)); + ADIO_Offset *sourceAggsForMyDataFDStart = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset)); + ADIO_Offset *sourceAggsForMyDataFDEnd = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset)); + int numSourceAggs = 0; + + /* compute number of rounds */ + ADIO_Offset numberOfRounds = (ADIO_Offset)((((ADIO_Offset)(end_offsets[nprocs-1]-st_offsets[0]))/((ADIO_Offset)((ADIO_Offset)coll_bufsize*(ADIO_Offset)naggs)))) + 1; + + /* This data structure holds the beginning offset and len list index for the range to be read + * coresponding to the round and source agg. + */ + int **sourceAggsForMyDataFirstOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *)); + for (i=0;i<numberOfRounds;i++) { + sourceAggsForMyDataFirstOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int)); + for (j=0;j<naggs;j++) + sourceAggsForMyDataFirstOffLenIndex[i][j] = -1; + } + + /* This data structure holds the ending offset and len list index for the range to be read + * coresponding to the round and source agg. + */ + int **sourceAggsForMyDataLastOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *)); + for (i=0;i<numberOfRounds;i++) + sourceAggsForMyDataLastOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int)); + + /* This data structure holds the base buffer offset for contiguous source data for the range to be read + * coresponding to the round and source agg. + */ + ADIO_Offset **baseRecvBufferOffset; + if (bufTypeIsContig) { + baseRecvBufferOffset = (ADIO_Offset **)ADIOI_Malloc(numberOfRounds * sizeof(ADIO_Offset *)); + for (i=0;i<numberOfRounds;i++) { + baseRecvBufferOffset[i] = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset)); + for (j=0;j<naggs;j++) + baseRecvBufferOffset[i][j] = -1; + } + } + ADIO_Offset currentRecvBufferOffset = 0; + + /* This data structure holds the number of extents, the index into the flattened buffer and the remnant length + * beyond the flattened buffer indice corresponding to the base buffer offset for non-contiguous source data + * for the range to be written coresponding to the round and target agg. + */ + NonContigSourceBufOffset **baseNonContigSourceBufferOffset; + if (!bufTypeIsContig) { + baseNonContigSourceBufferOffset = (NonContigSourceBufOffset **) ADIOI_Malloc(numberOfRounds * sizeof(NonContigSourceBufOffset *)); + for (i=0;i<numberOfRounds;i++) { + baseNonContigSourceBufferOffset[i] = (NonContigSourceBufOffset *)ADIOI_Malloc(naggs * sizeof(NonContigSourceBufOffset)); + /* initialize flatBufIndice to -1 to indicate that it is unset. + */ + for (j=0;j<naggs;j++) + baseNonContigSourceBufferOffset[i][j].flatBufIndice = -1; + } + } + + int currentDataTypeExtent = 0; + int currentFlatBufIndice=0; + ADIO_Offset currentRemLen = 0; + +#ifdef onesidedtrace + printf("NumberOfRounds is %d\n",numberOfRounds); + for (i=0;i<naggs;i++) + printf("fd->hints->ranklist[%d]is %d fd_start is %ld fd_end is %ld\n",i,fd->hints->ranklist[i],fd_start[i],fd_end[i]); + for (j=0;j<contig_access_count;j++) + printf("offset_list[%d] is %ld len_list is %ld\n",j,offset_list[j],len_list[j]); +#endif + + int currentAggRankListIndex = 0; + + /* This denotes the coll_bufsize boundaries within the source buffer for reading for 1 round. + */ + ADIO_Offset intraRoundCollBufsizeOffset = 0; + + /* This data structure tracks what source aggs need to be read to on what rounds. + */ + int *sourceAggsForMyDataCurrentRoundIter = (int *)ADIOI_Malloc(naggs * sizeof(int)); + for (i=0;i<naggs;i++) + sourceAggsForMyDataCurrentRoundIter[i] = 0; + + /* This is the first of the two main loops in this algorithm. The purpose of this loop is essentially to populate + * the data structures defined above for what source data blocks needs to go where (source agg) and when (round iter). + */ + int blockIter; + for (blockIter=0;blockIter<contig_access_count;blockIter++) { + + /* Determine the starting source buffer offset for this block - for iter 0 skip it since that value is 0. + */ + if (blockIter>0) { + if (bufTypeIsContig) { + currentRecvBufferOffset += len_list[blockIter-1]; + } + else { + /* Non-contiguous source datatype, count up the extents and indices to this point + * in the blocks. + */ + ADIO_Offset sourceBlockTotal = 0-currentRemLen; + while (sourceBlockTotal < len_list[blockIter-1]) { + maxNumContigOperations++; + sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + currentFlatBufIndice++; + if (currentFlatBufIndice == flatBuf->count) { + currentFlatBufIndice = 0; + currentDataTypeExtent++; + } + } + if (sourceBlockTotal > len_list[blockIter-1]) { + currentFlatBufIndice--; + if (currentFlatBufIndice < 0 ) { + currentDataTypeExtent--; + currentFlatBufIndice = flatBuf->count; + } + } + currentRemLen = len_list[blockIter-1] - sourceBlockTotal; +#ifdef onesidedtrace + printf("contig_access_count iter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); +#endif + } // !bufTypeIsContig + } // blockIter > 0 + + /* For the first iteration we need to include these maxNumContigOperations + * for non-contig case even though we did not need to compute the starting offset. + */ + if ((blockIter == 0) && (!bufTypeIsContig)) { + ADIO_Offset sourceBlockTotal = 0; + int tmpCurrentFlatBufIndice = currentFlatBufIndice; + while (sourceBlockTotal < len_list[0]) { + maxNumContigOperations++; + sourceBlockTotal += flatBuf->blocklens[tmpCurrentFlatBufIndice]; + tmpCurrentFlatBufIndice++; + if (tmpCurrentFlatBufIndice == flatBuf->count) { + tmpCurrentFlatBufIndice = 0; + } + } + } + + ADIO_Offset blockStart = offset_list[blockIter], blockLen = len_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1; + + /* Find the starting source agg for this block - normally it will be the current agg so guard the expensive + * while loop with a cheap if-check which for large numbers of small blocks will usually be false. + */ + if (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex]))) { + while (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex]))) + currentAggRankListIndex++; + }; + + /* Determine if this is a new source agg. + */ + if (blockIter>0) { + if ((offset_list[blockIter-1]+len_list[blockIter-1]-(ADIO_Offset)1) < fd_start[currentAggRankListIndex]) + numSourceAggs++; + } + + /* Determine which round to start reading. + */ + if ((blockStart - fd_start[currentAggRankListIndex]) >= coll_bufsize) { + ADIO_Offset currentRoundBlockStart = fd_start[currentAggRankListIndex]; + int startingRound = 0; + while (blockStart > (currentRoundBlockStart + coll_bufsize - (ADIO_Offset)1)) { + currentRoundBlockStart+=coll_bufsize; + startingRound++; + } + sourceAggsForMyDataCurrentRoundIter[numSourceAggs] = startingRound; + } + + /* Initialize the data structures if this is the first offset in the round/source agg. + */ + if (sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] == -1) { + sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex]; + sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[currentAggRankListIndex]; + sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex]; + sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; + if (bufTypeIsContig) + baseRecvBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = currentRecvBufferOffset; + else { + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + } + + intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + ((sourceAggsForMyDataCurrentRoundIter[numSourceAggs]+1) * coll_bufsize); +#ifdef onesidedtrace + printf("init settings numSourceAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld currentRecvBufferOffset set to %ld intraRoundCollBufsizeOffset set to %ld\n",numSourceAggs,blockIter,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter],currentRecvBufferOffset,intraRoundCollBufsizeOffset); +#endif + + } + + /* Replace the last offset block iter with this one. + */ + sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; + + /* If this blocks extends into the next file domain handle this situation. + */ + if (blockEnd > fd_end[currentAggRankListIndex]) { +#ifdef onesidedtrace + printf("large block, blockEnd %ld >= fd_end[currentAggRankListIndex] %ld\n",blockEnd,fd_end[currentAggRankListIndex]); +#endif + while (blockEnd >= fd_end[currentAggRankListIndex]) { + ADIO_Offset thisAggBlockEnd = fd_end[currentAggRankListIndex]; + if (thisAggBlockEnd >= intraRoundCollBufsizeOffset) { + while (thisAggBlockEnd >= intraRoundCollBufsizeOffset) { + sourceAggsForMyDataCurrentRoundIter[numSourceAggs]++; + intraRoundCollBufsizeOffset += coll_bufsize; + sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; + if (bufTypeIsContig) + baseRecvBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = currentRecvBufferOffset; + else { + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + } + + sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; +#ifdef onesidedtrace + printf("sourceAggsForMyDataCurrentRoundI%d] is now %d intraRoundCollBufsizeOffset is now %ld\n",numSourceAggs,sourceAggsForMyDataCurrentRoundIter[numSourceAggs],intraRoundCollBufsizeOffset); +#endif + } // while (thisAggBlockEnd >= intraRoundCollBufsizeOffset) + } // if (thisAggBlockEnd >= intraRoundCollBufsizeOffset) + + currentAggRankListIndex++; + + /* Skip over unused aggs. + */ + if (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) { + while (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) + currentAggRankListIndex++; + + } // (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) + + /* Start new source agg. + */ + if (blockEnd >= fd_start[currentAggRankListIndex]) { + numSourceAggs++; + sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex]; + sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[currentAggRankListIndex]; + sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex]; + sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; + if (bufTypeIsContig) + baseRecvBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = currentRecvBufferOffset; + else { + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + } + +#ifdef onesidedtrace + printf("large block init settings numSourceAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld\n",numSourceAggs,i,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter]); +#endif + intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + coll_bufsize; + sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; + } // if (blockEnd >= fd_start[currentAggRankListIndex]) + } // while (blockEnd >= fd_end[currentAggRankListIndex]) + } // if (blockEnd > fd_end[currentAggRankListIndex]) + + /* Else if we are still in the same file domain / source agg but have gone past the coll_bufsize and need + * to advance to the next round handle this situation. + */ + else if (blockEnd >= intraRoundCollBufsizeOffset) { + ADIO_Offset currentBlockEnd = blockEnd; + while (currentBlockEnd >= intraRoundCollBufsizeOffset) { + sourceAggsForMyDataCurrentRoundIter[numSourceAggs]++; + intraRoundCollBufsizeOffset += coll_bufsize; + sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; + if (bufTypeIsContig) + baseRecvBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = currentRecvBufferOffset; + else { + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].flatBufIndice = currentFlatBufIndice; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].dataTypeExtent = currentDataTypeExtent; + baseNonContigSourceBufferOffset[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs].remLen = currentRemLen; + } + sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter; +#ifdef onesidedtrace + printf("block less than fd currentBlockEnd is now %ld intraRoundCollBufsizeOffset is now %ld sourceAggsForMyDataCurrentRoundIter[%d] is now %d\n",currentBlockEnd, intraRoundCollBufsizeOffset, numSourceAggs,sourceAggsForMyDataCurrentRoundIter[numSourceAggs]); +#endif + } // while (currentBlockEnd >= intraRoundCollBufsizeOffset) + } // else if (blockEnd >= intraRoundCollBufsizeOffset) + + /* Need to advance numSourceAggs if this is the last source offset to + * include this one. + */ + if (blockIter == (contig_access_count-1)) + numSourceAggs++; + } + +#ifdef onesidedtrace + printf("numSourceAggs is %d\n",numSourceAggs); + for (i=0;i<numSourceAggs;i++) { + for (j=0;j<=sourceAggsForMyDataCurrentRoundIter[i];j++) + printf("sourceAggsForMyData[%d] is %d sourceAggsForMyDataFDStart[%d] is %ld sourceAggsForMyDataFDEnd is %ld sourceAggsForMyDataFirstOffLenIndex is %d with value %ld sourceAggsForMyDataLastOffLenIndex is %d with value %ld\n",i,sourceAggsForMyData[i],i,sourceAggsForMyDataFDStart[i],sourceAggsForMyDataFDEnd[i],sourceAggsForMyDataFirstOffLenIndex[j][i],offset_list[sourceAggsForMyDataFirstOffLenIndex[j][i]],sourceAggsForMyDataLastOffLenIndex[j][i],offset_list[sourceAggsForMyDataLastOffLenIndex[j][i]]); + + } +#endif + + ADIOI_Free(sourceAggsForMyDataCurrentRoundIter); + + /* use the two-phase buffer allocated in the file_open - no app should ever + * be both reading and reading at the same time */ + char *read_buf0 = fd->io_buf; + char *read_buf1 = fd->io_buf + coll_bufsize; + /* if threaded i/o selected, we'll do a kind of double buffering */ + char *read_buf = read_buf0; + + int currentReadBuf = 0; + int useIOBuffer = 0; +#ifdef ROMIO_GPFS + if (gpfsmpio_pthreadio && (numberOfRounds>1)) { + useIOBuffer = 1; + io_thread = pthread_self(); + } +#endif + + MPI_Win read_buf_window = fd->io_buf_window; + +#ifdef ACTIVE_TARGET + MPI_Win_fence(0, read_buf_window); +#endif + + ADIO_Offset currentRoundFDStart = 0, nextRoundFDStart = 0; + ADIO_Offset currentRoundFDEnd = 0, nextRoundFDEnd = 0; + + if (iAmUsedAgg) { + currentRoundFDStart = fd_start[myAggRank]; + nextRoundFDStart = fd_start[myAggRank]; + if (myAggRank == smallestFileDomainAggRank) { + if (currentRoundFDStart < firstFileOffset) + currentRoundFDStart = firstFileOffset; + if (nextRoundFDStart < firstFileOffset) + nextRoundFDStart = firstFileOffset; + } + } + +#ifdef ROMIO_GPFS + endTimeBase = MPI_Wtime(); + gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SETUP] += (endTimeBase-startTimeBase); + startTimeBase = MPI_Wtime(); +#endif + + + ADIO_Offset currentBaseRecvBufferOffset = 0; + + /* This is the second main loop of the algorithm, actually nested loop of source aggs within rounds. There are 2 flavors of this. + * For gpfsmpio_aggmethod of 1 each nested iteration for the source agg does an mpi_put on a contiguous chunk using a primative datatype + * determined using the data structures from the first main loop. For gpfsmpio_aggmethod of 2 each nested iteration for the source agg + * builds up data to use in created a derived data type for 1 mpi_put that is done for the source agg for each round. + */ + int roundIter; + for (roundIter=0;roundIter<numberOfRounds;roundIter++) { + + /* determine what offsets define the portion of the file domain the agg is reading this round */ + if (iAmUsedAgg) { + + currentRoundFDStart = nextRoundFDStart; + + if (!useIOBuffer || (roundIter == 0)) { + int amountDataToReadThisRound; + if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) { + currentRoundFDEnd = fd_end[myAggRank]; + amountDataToReadThisRound = ((currentRoundFDEnd-currentRoundFDStart)+1); + } + else { + currentRoundFDEnd = currentRoundFDStart + coll_bufsize - 1; + amountDataToReadThisRound = coll_bufsize; + } + + /* read currentRoundFDEnd bytes */ + ADIO_ReadContig(fd, read_buf,amountDataToReadThisRound, + MPI_BYTE, ADIO_EXPLICIT_OFFSET, currentRoundFDStart, + &status, error_code); + currentReadBuf = 1; + + } + if (useIOBuffer) { /* use the thread reader for the next round */ + /* switch back and forth between the read buffers so that the data aggregation code is diseminating 1 buffer while the thread is reading into the other */ + + if (roundIter > 0) + currentRoundFDEnd = nextRoundFDEnd; + + if (roundIter < (numberOfRounds-1)) { + nextRoundFDStart += coll_bufsize; + int amountDataToReadNextRound; + if ((fd_end[myAggRank] - nextRoundFDStart) < coll_bufsize) { + nextRoundFDEnd = fd_end[myAggRank]; + amountDataToReadNextRound = ((nextRoundFDEnd-nextRoundFDStart)+1); + } + else { + nextRoundFDEnd = nextRoundFDStart + coll_bufsize - 1; + amountDataToReadNextRound = coll_bufsize; + } + + if(!pthread_equal(io_thread, pthread_self())) { + pthread_join(io_thread, &thread_ret); + *error_code = *(int *)thread_ret; + if (*error_code != MPI_SUCCESS) return; + io_thread = pthread_self(); + + } + io_thread_args.fd = fd; + /* do a little pointer shuffling: background I/O works from one + * buffer while two-phase machinery fills up another */ + + if (currentReadBuf == 0) { + io_thread_args.buf = read_buf0; + currentReadBuf = 1; + read_buf = read_buf1; + } + else { + io_thread_args.buf = read_buf1; + currentReadBuf = 0; + read_buf = read_buf0; + } + io_thread_args.io_kind = ADIOI_READ; + io_thread_args.size = amountDataToReadNextRound; + io_thread_args.offset = nextRoundFDStart; + io_thread_args.status = status; + io_thread_args.error_code = *error_code; + if ( (pthread_create(&io_thread, NULL, + ADIOI_IO_Thread_Func, &(io_thread_args))) != 0) + io_thread = pthread_self(); + + } + else { /* last round */ + + if(!pthread_equal(io_thread, pthread_self())) { + pthread_join(io_thread, &thread_ret); + *error_code = *(int *)thread_ret; + if (*error_code != MPI_SUCCESS) return; + io_thread = pthread_self(); + + } + if (currentReadBuf == 0) { + read_buf = read_buf0; + } + else { + read_buf = read_buf1; + } + + } + } /* useIOBuffer */ + } /* IAmUsedAgg */ + else if (useIOBuffer) { + if (roundIter < (numberOfRounds-1)) { + if (currentReadBuf == 0) { + currentReadBuf = 1; + read_buf = read_buf1; + } + else { + currentReadBuf = 0; + read_buf = read_buf0; + } + } + else { + if (currentReadBuf == 0) { + read_buf = read_buf0; + } + else { + read_buf = read_buf1; + } + } + + } + // wait until the read buffers are full before we start pulling from the source procs + MPI_Barrier(fd->comm); + + int aggIter; + for (aggIter=0;aggIter<numSourceAggs;aggIter++) { + + /* If we have data for the round/agg process it. + */ + if ((bufTypeIsContig && (baseRecvBufferOffset[roundIter][aggIter] != -1)) || (!bufTypeIsContig && (baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice != -1))) { + + ADIO_Offset currentRoundFDStartForMySourceAgg = (ADIO_Offset)((ADIO_Offset)sourceAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)roundIter*(ADIO_Offset)coll_bufsize)); + ADIO_Offset currentRoundFDEndForMySourceAgg = (ADIO_Offset)((ADIO_Offset)sourceAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)(roundIter+1)*(ADIO_Offset)coll_bufsize) - (ADIO_Offset)1); + + int sourceAggContigAccessCount = 0; + + /* These data structures are used for the derived datatype mpi_put + * in the gpfsmpio_aggmethod of 2 case. + */ + int *sourceAggBlockLengths; + MPI_Aint *sourceAggDisplacements, *recvBufferDisplacements; + MPI_Datatype *sourceAggDataTypes; + + int allocatedDerivedTypeArrays = 0; + + ADIO_Offset recvBufferOffset = 0; + + /* Process the range of offsets for this source agg. + */ + int offsetIter; + for (offsetIter=sourceAggsForMyDataFirstOffLenIndex[roundIter][aggIter];offsetIter<=sourceAggsForMyDataLastOffLenIndex[roundIter][aggIter];offsetIter++) { + if (currentRoundFDEndForMySourceAgg > sourceAggsForMyDataFDEnd[aggIter]) + currentRoundFDEndForMySourceAgg = sourceAggsForMyDataFDEnd[aggIter]; + + ADIO_Offset offsetStart = offset_list[offsetIter], offsetEnd = (offset_list[offsetIter]+len_list[offsetIter]-(ADIO_Offset)1); + + /* Get the base source buffer offset for this iterataion of the source agg in the round. + * For the first one just get the predetermined base value, for subsequent ones keep track + * of the value for each iteration and increment it. + */ + if (offsetIter == sourceAggsForMyDataFirstOffLenIndex[roundIter][aggIter]) { + if (bufTypeIsContig) + currentBaseRecvBufferOffset = baseRecvBufferOffset[roundIter][aggIter]; + else { + currentFlatBufIndice = baseNonContigSourceBufferOffset[roundIter][aggIter].flatBufIndice; + currentDataTypeExtent = baseNonContigSourceBufferOffset[roundIter][aggIter].dataTypeExtent; + currentRemLen = baseNonContigSourceBufferOffset[roundIter][aggIter].remLen; +#ifdef onesidedtrace + printf("currentFlatBufIndice initially set to %d starting this round/agg %d/%d\n",currentFlatBufIndice,roundIter,aggIter); +#endif + } + } + else { + if (bufTypeIsContig) + currentBaseRecvBufferOffset += len_list[offsetIter-1]; + else { + + /* For non-contiguous source datatype count up the extents and indices to this point. + */ + ADIO_Offset sourceBlockTotal = 0-currentRemLen; + while (sourceBlockTotal < len_list[offsetIter-1]) { + sourceBlockTotal += flatBuf->blocklens[currentFlatBufIndice]; + currentFlatBufIndice++; + if (currentFlatBufIndice == flatBuf->count) { + currentFlatBufIndice = 0; + currentDataTypeExtent++; + } + } // while + if (sourceBlockTotal > len_list[offsetIter-1]) + currentFlatBufIndice--; + currentRemLen = len_list[offsetIter-1] - sourceBlockTotal; +#ifdef onesidedtrace + printf("contig_access_count source agg %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentRemLen is now %ld\n",aggIter,currentFlatBufIndice,currentDataTypeExtent,currentRemLen); +#endif + } + + } + + /* For source buffer contiguous here is the offset we will use for this iteration. + */ + if (bufTypeIsContig) + recvBufferOffset = currentBaseRecvBufferOffset; + +#ifdef onesidedtrace + printf("roundIter %d source iter %d sourceAggsForMyData is %d offset_list[%d] is %ld len_list[%d] is %ld sourceAggsForMyDataFDStart is %ld sourceAggsForMyDataFDEnd is %ld currentRoundFDStartForMySourceAgg is %ld currentRoundFDEndForMySourceAgg is %ld sourceAggsForMyDataFirstOffLenIndex is %d baseRecvBufferOffset is %ld\n", + roundIter,aggIter,sourceAggsForMyData[aggIter],offsetIter,offset_list[offsetIter],offsetIter,len_list[offsetIter], + sourceAggsForMyDataFDStart[aggIter],sourceAggsForMyDataFDEnd[aggIter], + currentRoundFDStartForMySourceAgg,currentRoundFDEndForMySourceAgg,sourceAggsForMyDataFirstOffLenIndex[roundIter][aggIter],baseRecvBufferOffset[roundIter][aggIter]); +#endif + + + /* Determine the amount of data and exact source buffer offsets to use. + */ + int bufferAmountToRecv = 0; + + if ((offsetStart >= currentRoundFDStartForMySourceAgg) && (offsetStart <= currentRoundFDEndForMySourceAgg)) { + if (offsetEnd > currentRoundFDEndForMySourceAgg) + bufferAmountToRecv = (currentRoundFDEndForMySourceAgg - offsetStart) +1; + else + bufferAmountToRecv = (offsetEnd - offsetStart) +1; + } + else if ((offsetEnd >= currentRoundFDStartForMySourceAgg) && (offsetEnd <= currentRoundFDEndForMySourceAgg)) { + if (offsetEnd > currentRoundFDEndForMySourceAgg) + bufferAmountToRecv = (currentRoundFDEndForMySourceAgg - currentRoundFDStartForMySourceAgg) +1; + else + bufferAmountToRecv = (offsetEnd - currentRoundFDStartForMySourceAgg) +1; + if (offsetStart < currentRoundFDStartForMySourceAgg) { + if (bufTypeIsContig) + recvBufferOffset += (currentRoundFDStartForMySourceAgg-offsetStart); + offsetStart = currentRoundFDStartForMySourceAgg; + } + } + else if ((offsetStart <= currentRoundFDStartForMySourceAgg) && (offsetEnd >= currentRoundFDEndForMySourceAgg)) { + bufferAmountToRecv = (currentRoundFDEndForMySourceAgg - currentRoundFDStartForMySourceAgg) +1; + if (bufTypeIsContig) + recvBufferOffset += (currentRoundFDStartForMySourceAgg-offsetStart); + offsetStart = currentRoundFDStartForMySourceAgg; + } + + if (bufferAmountToRecv > 0) { /* we have data to recv this round */ + if (gpfsmpio_aggmethod == 2) { + /* Only allocate these arrays if we are using method 2 and only do it once for this round/source agg. + */ + if (!allocatedDerivedTypeArrays) { + sourceAggBlockLengths = (int *)ADIOI_Malloc(maxNumContigOperations * sizeof(int)); + sourceAggDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint)); + recvBufferDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint)); + sourceAggDataTypes = (MPI_Datatype *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Datatype)); + allocatedDerivedTypeArrays = 1; + } + } + + /* For the non-contiguous source datatype we need to determine the number,size and offsets of the chunks + * from the source buffer to be sent to this contiguous chunk defined by the round/agg/iter in the source. + */ + int numNonContigSourceChunks = 0; + ADIO_Offset *nonContigSourceOffsets; + int *nonContigSourceLens; + ADIO_Offset baseDatatypeInstanceOffset = 0; + + if (!bufTypeIsContig) { + + currentRecvBufferOffset = (ADIO_Offset)((ADIO_Offset)currentDataTypeExtent * (ADIO_Offset)bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentRemLen; +#ifdef onesidedtrace + printf("!bufTypeIsContig currentRecvBufferOffset set to %ld for roundIter %d source %d currentDataTypeExtent %d flatBuf->indices[currentFlatBufIndice] %ld currentRemLen %ld currentFlatBufIndice %d\n",currentRecvBufferOffset,roundIter,aggIter,currentDataTypeExtent,flatBuf->indices[currentFlatBufIndice],currentRemLen,currentFlatBufIndice); +#endif + + /* Count the chunks first to see how much to malloc from the ending point from above code. + */ + int recvBytesCounted = 0; + int tmpFlatBufIndice = currentFlatBufIndice; + int maxNumNonContigSourceChunks = 2; // over-initialize for potential remnants on both ends + + while (recvBytesCounted < bufferAmountToRecv) { + maxNumNonContigSourceChunks++; + if (tmpFlatBufIndice == flatBuf->count) { + tmpFlatBufIndice = 0; + } + recvBytesCounted += flatBuf->blocklens[tmpFlatBufIndice]; + tmpFlatBufIndice++; + } + + nonContigSourceOffsets = (ADIO_Offset *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(ADIO_Offset)); + nonContigSourceLens = (int *)ADIOI_Malloc(maxNumNonContigSourceChunks * sizeof(int)); + + /* now populate the nonContigSourceOffsets and nonContigSourceLens arrays for use in the one-sided operations. + */ + int ncArrayIndex = 0; + int remainingBytesToLoadedIntoNCArrays = bufferAmountToRecv; + ADIO_Offset indexIntoCurrentIndice = 0; + if (currentRemLen > 0) + indexIntoCurrentIndice = flatBuf->blocklens[currentFlatBufIndice] - currentRemLen; + + int datatypeInstances = currentDataTypeExtent; + tmpFlatBufIndice = currentFlatBufIndice; + while (remainingBytesToLoadedIntoNCArrays > 0) { + nonContigSourceOffsets[ncArrayIndex] = currentRecvBufferOffset; + + if ((flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice) > remainingBytesToLoadedIntoNCArrays) { + nonContigSourceLens[ncArrayIndex] = remainingBytesToLoadedIntoNCArrays; + remainingBytesToLoadedIntoNCArrays = 0; + } + else { + nonContigSourceLens[ncArrayIndex] = (int)(flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); + remainingBytesToLoadedIntoNCArrays -= (flatBuf->blocklens[tmpFlatBufIndice] - indexIntoCurrentIndice); + } + indexIntoCurrentIndice = 0; // only worry about beginning remnant for first iter + + tmpFlatBufIndice++; + if (tmpFlatBufIndice == flatBuf->count) { + tmpFlatBufIndice = 0; + datatypeInstances++; + baseDatatypeInstanceOffset = datatypeInstances * bufTypeExtent; + } + currentRecvBufferOffset = baseDatatypeInstanceOffset + flatBuf->indices[tmpFlatBufIndice]; + ncArrayIndex++; + numNonContigSourceChunks++; + } // while +#ifdef onesidedtrace + printf("currentRecvBufferOffset finally set to %ld\n",currentRecvBufferOffset); +#endif + } // !bufTypeIsContig + + /* Determine the offset into the source window. + */ + MPI_Aint sourceDisplacementToUseThisRound = (MPI_Aint) ((ADIO_Offset)offsetStart - currentRoundFDStartForMySourceAgg); + + /* If using the thread readr select the appropriate side of the split window. + */ + if (useIOBuffer && (read_buf == read_buf1)) { + sourceDisplacementToUseThisRound += coll_bufsize; + } + + /* For gpfsmpio_aggmethod of 1 do the mpi_put using the primitive MPI_BYTE type on each contiguous chunk of source data. + */ + if (gpfsmpio_aggmethod == 1) { +#ifndef ACTIVE_TARGET + MPI_Win_lock(MPI_LOCK_SHARED, sourceAggsForMyData[aggIter], 0, read_buf_window); +#endif + if (bufTypeIsContig) { + MPI_Get(&((char*)buf)[recvBufferOffset],bufferAmountToRecv, MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, bufferAmountToRecv,MPI_BYTE,read_buf_window); + } + else { + for (i=0;i<numNonContigSourceChunks;i++) { + MPI_Get(&((char*)buf)[nonContigSourceOffsets[i]],nonContigSourceLens[i], MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, nonContigSourceLens[i],MPI_BYTE,read_buf_window); +#ifdef onesidedtrace + printf("mpi_put[%d] nonContigSourceOffsets is %d of nonContigSourceLens %d to source disp %d\n",i,nonContigSourceOffsets[i],nonContigSourceLens[i],sourceDisplacementToUseThisRound); +#endif + sourceDisplacementToUseThisRound += nonContigSourceLens[i]; + } + } +#ifndef ACTIVE_TARGET + MPI_Win_unlock(sourceAggsForMyData[aggIter], read_buf_window); +#endif + + } + + /* For gpfsmpio_aggmethod of 2 populate the data structures for this round/agg for this offset iter + * to be used subsequently when building the derived type for 1 mpi_put for all the data for this + * round/agg. + */ + else if (gpfsmpio_aggmethod == 2) { + + if (bufTypeIsContig) { + sourceAggBlockLengths[sourceAggContigAccessCount]= bufferAmountToRecv; + sourceAggDataTypes[sourceAggContigAccessCount] = MPI_BYTE; + sourceAggDisplacements[sourceAggContigAccessCount] = sourceDisplacementToUseThisRound; + recvBufferDisplacements[sourceAggContigAccessCount] = (MPI_Aint)recvBufferOffset; + sourceAggContigAccessCount++; + } + else { + for (i=0;i<numNonContigSourceChunks;i++) { + sourceAggBlockLengths[sourceAggContigAccessCount]= nonContigSourceLens[i]; + sourceAggDataTypes[sourceAggContigAccessCount] = MPI_BYTE; + sourceAggDisplacements[sourceAggContigAccessCount] = sourceDisplacementToUseThisRound; + recvBufferDisplacements[sourceAggContigAccessCount] = (MPI_Aint)nonContigSourceOffsets[i]; +#ifdef onesidedtrace + printf("mpi_put building arrays for iter %d - recvBufferDisplacements is %d of nonContigSourceLens %d to source disp %d sourceAggContigAccessCount is %d sourceAggBlockLengths[sourceAggContigAccessCount] is %d\n",i,recvBufferDisplacements[sourceAggContigAccessCount],nonContigSourceLens[i],sourceAggDisplacements[sourceAggContigAccessCount],sourceAggContigAccessCount, sourceAggBlockLengths[sourceAggContigAccessCount]); +#endif + sourceAggContigAccessCount++; + sourceDisplacementToUseThisRound += nonContigSourceLens[i]; + } + } + } +#ifdef onesidedtrace + printf("roundIter %d bufferAmountToRecv is %d recvBufferOffset is %d offsetStart is %ld currentRoundFDStartForMySourceAgg is %ld sourceDisplacementToUseThisRound is %ld sourceAggsForMyDataFDStart[aggIter] is %ld\n",roundIter, bufferAmountToRecv,recvBufferOffset, offsetStart,currentRoundFDStartForMySourceAgg,sourceDisplacementToUseThisRound,sourceAggsForMyDataFDStart[aggIter]); +#endif + + if (!bufTypeIsContig) { + ADIOI_Free(nonContigSourceOffsets); + ADIOI_Free(nonContigSourceLens); + } + } // bufferAmountToRecv > 0 + } // contig list + + /* For gpfsmpio_aggmethod of 2 now build the derived type using the data from this round/agg and do 1 single mpi_put. + */ + if (gpfsmpio_aggmethod == 2) { + MPI_Datatype recvBufferDerivedDataType, sourceBufferDerivedDataType; + MPI_Type_create_struct(sourceAggContigAccessCount, sourceAggBlockLengths, recvBufferDisplacements, sourceAggDataTypes, &recvBufferDerivedDataType); + MPI_Type_commit(&recvBufferDerivedDataType); + MPI_Type_create_struct(sourceAggContigAccessCount, sourceAggBlockLengths, sourceAggDisplacements, sourceAggDataTypes, &sourceBufferDerivedDataType); + MPI_Type_commit(&sourceBufferDerivedDataType); +// printf("round %d mpi_put of derived type to agg %d sourceAggContigAccessCount is %d\n",roundIter, sourceAggsForMyData[aggIter],sourceAggContigAccessCount); +#ifndef ACTIVE_TARGET + MPI_Win_lock(MPI_LOCK_SHARED, sourceAggsForMyData[aggIter], 0, read_buf_window); +#endif + MPI_Get(((char*)buf),1, recvBufferDerivedDataType,sourceAggsForMyData[aggIter],0, 1,sourceBufferDerivedDataType,read_buf_window); + +#ifndef ACTIVE_TARGET + MPI_Win_unlock(sourceAggsForMyData[aggIter], read_buf_window); +#endif + + if (allocatedDerivedTypeArrays) { + ADIOI_Free(sourceAggBlockLengths); + ADIOI_Free(sourceAggDisplacements); + ADIOI_Free(sourceAggDataTypes); + ADIOI_Free(recvBufferDisplacements); + } + MPI_Type_free(&recvBufferDerivedDataType); + MPI_Type_free(&sourceBufferDerivedDataType); + } + } // baseoffset != -1 + } // source aggs + + /* the source procs recv the requested data to the aggs */ + +#ifdef ACTIVE_TARGET + MPI_Win_fence(0, read_buf_window); +#else + MPI_Barrier(fd->comm); +#endif + + nextRoundFDStart = currentRoundFDStart + coll_bufsize; + + } /* for-loop roundIter */ + +#ifdef ROMIO_GPFS + endTimeBase = MPI_Wtime(); + gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase); +#endif + + if (useIOBuffer) { /* thread readr cleanup */ + + if ( !pthread_equal(io_thread, pthread_self()) ) { + pthread_join(io_thread, &thread_ret); + *error_code = *(int *)thread_ret; + } + + } + + ADIOI_Free(sourceAggsForMyData); + ADIOI_Free(sourceAggsForMyDataFDStart); + ADIOI_Free(sourceAggsForMyDataFDEnd); + + for (i=0;i<numberOfRounds;i++) { + ADIOI_Free(sourceAggsForMyDataFirstOffLenIndex[i]); + ADIOI_Free(sourceAggsForMyDataLastOffLenIndex[i]); + if (bufTypeIsContig) + ADIOI_Free(baseRecvBufferOffset[i]); + else + ADIOI_Free(baseNonContigSourceBufferOffset[i]); + } + ADIOI_Free(sourceAggsForMyDataFirstOffLenIndex); + ADIOI_Free(sourceAggsForMyDataLastOffLenIndex); + if (bufTypeIsContig) + ADIOI_Free(baseRecvBufferOffset); + else + ADIOI_Free(baseNonContigSourceBufferOffset); + + return; +} ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c | 15 + src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c | 47 + src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h | 5 +- src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c | 28 + src/mpi/romio/adio/common/Makefile.mk | 1 + src/mpi/romio/adio/common/ad_close.c | 2 +- src/mpi/romio/adio/common/ad_open.c | 6 +- src/mpi/romio/adio/common/onesided_aggregation.c | 2075 ++++++++++++++++++++++ src/mpi/romio/adio/include/adio.h | 4 + src/mpi/romio/adio/include/adioi.h | 24 + src/mpid/common/datatype/mpid_type_commit.c | 7 +- src/mpid/common/datatype/mpid_type_dup.c | 7 +- src/mpid/pamid/include/mpidi_datatypes.h | 1 + src/mpid/pamid/include/mpidpre.h | 8 + src/mpid/pamid/src/Makefile.mk | 3 +- src/mpid/pamid/src/mpid_init.c | 5 +- src/mpid/pamid/src/mpidi_env.c | 5 + src/mpid/pamid/src/mpidi_pami_datatype.c | 155 ++ src/mpid/pamid/src/onesided/mpid_win_get.c | 55 +- src/mpid/pamid/src/onesided/mpid_win_put.c | 53 +- src/mpid/pamid/src/onesided/mpidi_onesided.h | 2 + 21 files changed, 2491 insertions(+), 17 deletions(-) create mode 100644 src/mpi/romio/adio/common/onesided_aggregation.c create mode 100644 src/mpid/pamid/src/mpidi_pami_datatype.c hooks/post-receive -- MPICH primary repository

1 0