May 2014 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.1-288-g7e44a0f
by noreply＠mpich.org 31 May '14

31 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 7e44a0f14764b56823811ff8235ce802f5bdfae6 (commit) via 05eeccb59d7a0db0278e533b7b82cc059c514906 (commit) from 6063f89812b3b1b718abffa1b0c895cd5a19754c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/7e44a0f14764b56823811ff8235ce802f… commit 7e44a0f14764b56823811ff8235ce802f5bdfae6 Author: Huiwei Lu <huiweilu(a)mcs.anl.gov> Date: Wed Apr 30 23:10:57 2014 -0500 Adds two tests to MPI_Comm_idup Two multi-threaded tests are added for MPI_Comm_idup in test/mpi/threads/comm: ctxidup.c and comm_idup.c. The former has passed the test but the latter still not (marked in testlist as xfail). Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/test/mpi/.gitignore b/test/mpi/.gitignore index f3a1916..5966d8f 100644 --- a/test/mpi/.gitignore +++ b/test/mpi/.gitignore @@ -1031,7 +1031,9 @@ /threads/comm/comm_create_group_threads /threads/comm/comm_create_threads /threads/comm/comm_dup_deadlock +/threads/comm/comm_idup /threads/comm/ctxdup +/threads/comm/ctxidup /threads/comm/dup_leak_test /threads/init/initth /threads/pt2pt/threads diff --git a/test/mpi/threads/comm/Makefile.am b/test/mpi/threads/comm/Makefile.am index 09e3fe8..0fb58ad 100644 --- a/test/mpi/threads/comm/Makefile.am +++ b/test/mpi/threads/comm/Makefile.am @@ -9,5 +9,5 @@ include $(top_srcdir)/threads/Makefile_threads.mtest EXTRA_DIST = testlist -noinst_PROGRAMS = ctxdup dup_leak_test comm_dup_deadlock comm_create_threads comm_create_group_threads +noinst_PROGRAMS = ctxdup dup_leak_test comm_dup_deadlock comm_create_threads comm_create_group_threads comm_idup ctxidup diff --git a/test/mpi/threads/comm/comm_idup.c b/test/mpi/threads/comm/comm_idup.c new file mode 100644 index 0000000..1a183ee --- /dev/null +++ b/test/mpi/threads/comm/comm_idup.c @@ -0,0 +1,98 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2012 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#include <stdio.h> +#include <mpi.h> +#include "mpitest.h" +#include "mpithreadtest.h" + +#define NUM_THREADS 2 +#define NUM_ITER 1 + +#define check(X_) \ + do { \ + if (!(X_)) { \ + printf("[%s:%d] -- Assertion failed: %s\n", __FILE__, __LINE__, #X_);\ + MPI_Abort(MPI_COMM_WORLD, 1); \ + } \ + } while (0) + +MPI_Comm comms[NUM_THREADS]; +MTEST_THREAD_LOCK_TYPE comm_lock; +int rank, size; +int verbose = 0; + +MTEST_THREAD_RETURN_TYPE test_comm_dup(void *arg) +{ + int rank; + int i; + MPI_Request req; + + MPI_Comm_rank(comms[*(int*)arg], &rank); + + for (i = 0; i < NUM_ITER; i++) { + MPI_Comm comm, self_dup; + + if (*(int*)arg == rank) { + MTestSleep(1); + } + + MTest_thread_lock(&comm_lock); + if (verbose) printf("%d: Thread %d - COMM_IDUP %d start\n", rank, *(int*)arg, i); + MPI_Comm_idup(MPI_COMM_SELF, &self_dup, &req); + MPI_Wait(&req, MPI_STATUS_IGNORE); + if (verbose) printf("\t%d: Thread %d - COMM_IDUP %d finish\n", rank, *(int*)arg, i); + MTest_thread_unlock(&comm_lock); + MPI_Comm_free(&self_dup); + + if (verbose) printf("%d: Thread %d - comm_idup %d start\n", rank, *(int*)arg, i); + MPI_Comm_idup(comms[*(int*)arg], &comm, &req); + MPI_Wait(&req, MPI_STATUS_IGNORE); + MPI_Comm_free(&comm); + if (verbose) printf("\t%d: Thread %d - comm_idup %d finish\n", rank, *(int*)arg, i); + } + + if (verbose) printf("%d: Thread %d - Done.\n", rank, *(int*)arg); + return (MTEST_THREAD_RETURN_TYPE)0; +} + + +int main(int argc, char **argv) +{ + int thread_args[NUM_THREADS]; + int i, provided; + + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + + check(provided == MPI_THREAD_MULTIPLE); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + MTest_thread_lock_create(&comm_lock); + + for (i = 0; i < NUM_THREADS; i++) { + MPI_Comm_dup(MPI_COMM_WORLD, &comms[i]); + } + + for (i = 0; i < NUM_THREADS; i++) { + thread_args[i] = i; + MTest_Start_thread( test_comm_dup, (void *)&thread_args[i] ); + } + + MTest_Join_threads(); + + for (i = 0; i < NUM_THREADS; i++) { + MPI_Comm_free(&comms[i]); + } + + if (rank == 0) + printf(" No Errors\n"); + + MPI_Finalize(); + + return 0; +} diff --git a/test/mpi/threads/comm/ctxidup.c b/test/mpi/threads/comm/ctxidup.c new file mode 100644 index 0000000..b13fd91 --- /dev/null +++ b/test/mpi/threads/comm/ctxidup.c @@ -0,0 +1,104 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * + * (C) 2006 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ +#include "mpi.h" +#include <stdio.h> +#include <stdlib.h> +#include "mpitest.h" +#include "mpithreadtest.h" + +/* + * static char MTEST_Descrip[] = "Creating communications concurrently in different threads"; + */ + +MTEST_THREAD_RETURN_TYPE dup_thread(void *); + +MTEST_THREAD_RETURN_TYPE dup_thread(void *p) +{ + int rank; + int buffer[1]; + MPI_Comm *comm2_ptr = (MPI_Comm *)p; + MPI_Comm comm3; + MPI_Request req; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + if (rank & 0x1) { + /* If odd, wait for message */ + MPI_Recv( buffer, 0, MPI_INT, rank, 0, MPI_COMM_WORLD, + MPI_STATUS_IGNORE ); + } + MPI_Comm_idup( *comm2_ptr, &comm3, &req ); + MPI_Wait(&req, MPI_STATUS_IGNORE); + + MPI_Barrier( comm3 ); + MPI_Recv( buffer, 0, MPI_INT, rank, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE ); + + MPI_Comm_free( &comm3 ); + /* Tell the main thread that we're done */ + MPI_Send( buffer, 0, MPI_INT, rank, 2, MPI_COMM_WORLD ); + + return (MTEST_THREAD_RETURN_TYPE)0; +} + +int main( int argc, char *argv[] ) +{ + int rank, size; + int provided; + int buffer[1]; + MPI_Comm comm1, comm2, comm4; + + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + /* Check that we're multi-threaded */ + if (provided != MPI_THREAD_MULTIPLE) { + if (rank == 0) { + printf("MPI_Init_thread must return MPI_THREAD_MULTIPLE in order for this test to run.\n"); + fflush(stdout); + } + MPI_Finalize(); + return 1; + } + + /* The test is this: + The main thread on ODD processors tells the other thread to start + a comm dup(on comm2), then starts a comm dup(on comm1) after a delay. + The main thread on even processors starts a comm dup(on comm1) + + The second thread on ODD processors waits until it gets a message + (from the same process) before starting the comm dup on comm2. + */ + + /* Create two communicators */ + MPI_Comm_dup( MPI_COMM_WORLD, &comm1 ); + MPI_Comm_dup( MPI_COMM_WORLD, &comm2 ); + + /* Start a thread that will perform a dup comm2 */ + MTest_Start_thread( dup_thread, (void *)&comm2 ); + + /* If we're odd, send to our new thread and then delay */ + if (rank & 0x1) { + MPI_Ssend( buffer, 0, MPI_INT, rank, 0, MPI_COMM_WORLD ); + MTestSleep(1); + } + MPI_Comm_dup( comm1, &comm4 ); + + /* Tell the threads to exit after we've created our new comm */ + MPI_Barrier( comm4 ); + MPI_Ssend( buffer, 0, MPI_INT, rank, 1, MPI_COMM_WORLD ); + MPI_Recv( buffer, 0, MPI_INT, rank, 2, MPI_COMM_WORLD, MPI_STATUS_IGNORE ); + + MPI_Comm_free( &comm4 ); + MPI_Comm_free( &comm1 ); + MPI_Comm_free( &comm2 ); + + MTest_Finalize(0); + MPI_Finalize(); + return 0; +} diff --git a/test/mpi/threads/comm/testlist b/test/mpi/threads/comm/testlist index 866a3f1..6d94573 100644 --- a/test/mpi/threads/comm/testlist +++ b/test/mpi/threads/comm/testlist @@ -3,3 +3,5 @@ dup_leak_test 2 comm_dup_deadlock 4 comm_create_threads 4 comm_create_group_threads 4 +comm_idup 4 mpiversion=3.0 xfail=ticket2069 +ctxidup 4 http://git.mpich.org/mpich.git/commitdiff/05eeccb59d7a0db0278e533b7b82cc059… commit 05eeccb59d7a0db0278e533b7b82cc059c514906 Author: Huiwei Lu <huiweilu(a)mcs.anl.gov> Date: Tue Apr 1 23:01:01 2014 -0500 Fixes MPI_Comm_idup This patch fixes two related tickets: 1. MPI_Comm_idup in multithreaded environments 2. MPI_Comm_idup fails to create multiple communicators Because these two tickets are tightly coupled, so they are fixed in this single patch. The original code did not consider the multithreaded case and did not use progress engine in correct order when saving a copy of global mask to local thread. Following changes were made to implement the MPI_Comm_idup correctly: 1. It shares the same global flag 'mask_in_use' with other communicator functions to protect access to context_mask. And use CONTEXTID lock to protext critical sections. 2. It uses the same algorithm as multithreaded MPI_Comm_dup (multi-threaded vertion of MPIR_Get_contextid_sparse_group) to allocate a context id, but in a nonblocking way. In the case of conflicts, the algorithm needs to retry the allocation process again. In the nonblocking algorithm, 1) new entries are inserted to the end of schedule to replace the 'while' loop in MPI_Comm_dup algorithm; 2) all arguments passed to sched_get_cid_nonblock are saved to gcn_state in order to be called in the future; 3) in sched_cb_gcn_allocate_cid, if the first try failed, it will insert sched_cb_gcn_copy_mask to the schedule again. 3. There is a subtle difference between INTRACOMM and INTERCOMM when duplicating a communicator. They needed to be treated differently in current algorithm. Specifically, 1) when calling sched_get_cid_nonblock, the parameters are different; 2) updating newcommp->recvcontext_id in MPIR_Get_intercomm_contextid_nonblock has been moved to sched_cb_gcn_bcast because this should happen after sched_cb_gcn_allocate_cid has succeed. Fixes #1935 Fixes #1913 Signed-off-by: Pavan Balaji <balaji(a)anl.gov> diff --git a/src/mpi/comm/commutil.c b/src/mpi/comm/commutil.c index 1651fce..7d29e03 100644 --- a/src/mpi/comm/commutil.c +++ b/src/mpi/comm/commutil.c @@ -1222,57 +1222,166 @@ fn_fail: struct gcn_state { MPIR_Context_id_t *ctx0; MPIR_Context_id_t *ctx1; + int own_mask; + MPID_Comm *comm_ptr; + MPID_Comm *comm_ptr_inter; + MPID_Sched_t s; + MPID_Comm_kind_t gcn_cid_kind; uint32_t local_mask[MPIR_MAX_CONTEXT_MASK]; }; +static int sched_cb_gcn_copy_mask(MPID_Comm *comm, int tag, void *state); +static int sched_cb_gcn_allocate_cid(MPID_Comm *comm, int tag, void *state); +static int sched_cb_gcn_bcast(MPID_Comm *comm, int tag, void *state); + +#undef FUNCNAME +#define FUNCNAME sched_cb_gcn_bcast +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +static int sched_cb_gcn_bcast(MPID_Comm *comm, int tag, void *state) +{ + int mpi_errno = MPI_SUCCESS; + struct gcn_state *st = state; + + if (st->gcn_cid_kind == MPID_INTERCOMM) { + if (st->comm_ptr_inter->rank == 0) { + mpi_errno = MPID_Sched_recv(st->ctx1, 1, MPIR_CONTEXT_ID_T_DATATYPE, 0, st->comm_ptr_inter, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + mpi_errno = MPID_Sched_send(st->ctx0, 1, MPIR_CONTEXT_ID_T_DATATYPE, 0, st->comm_ptr_inter, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPID_SCHED_BARRIER(st->s); + } + + mpi_errno = st->comm_ptr->coll_fns->Ibcast_sched(st->ctx1, 1, + MPIR_CONTEXT_ID_T_DATATYPE, 0, st->comm_ptr, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPID_SCHED_BARRIER(st->s); + } + + mpi_errno = MPID_Sched_cb(&MPIR_Sched_cb_free_buf, st, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + +fn_fail: + return mpi_errno; +} + #undef FUNCNAME -#define FUNCNAME gcn_helper +#define FUNCNAME sched_cb_gcn_allocate_cid #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -static int gcn_helper(MPID_Comm *comm, int tag, void *state) +static int sched_cb_gcn_allocate_cid(MPID_Comm *comm, int tag, void *state) { int mpi_errno = MPI_SUCCESS; struct gcn_state *st = state; MPIR_Context_id_t newctxid; - newctxid = MPIR_Find_and_allocate_context_id(st->local_mask); - if (!newctxid) { - int nfree = -1; - int ntotal = -1; - MPIR_ContextMaskStats(&nfree, &ntotal); - MPIU_ERR_SETANDJUMP3(mpi_errno, MPI_ERR_OTHER, - "**toomanycomm", "**toomanycomm %d %d %d", - nfree, ntotal, /*ignore_id=*/0); + MPIU_THREAD_CS_ENTER(CONTEXTID,); + if (st->own_mask) { + newctxid = MPIR_Find_and_allocate_context_id(st->local_mask); + + if (st->ctx0) + *st->ctx0 = newctxid; + if (st->ctx1) + *st->ctx1 = newctxid; + mask_in_use = 0; + + if (newctxid > 0) { + if (lowestContextId == st->comm_ptr->context_id) + lowestContextId = MPIR_MAXID; + } + } + MPIU_THREAD_CS_EXIT(CONTEXTID,); + + if (*st->ctx0 == 0) { + /* do not own mask, try again */ + mpi_errno = MPID_Sched_cb(&sched_cb_gcn_copy_mask, st, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPID_SCHED_BARRIER(st->s); + } else { + /* Successfully allocated a context id */ + mpi_errno = MPID_Sched_cb(&sched_cb_gcn_bcast, st, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPID_SCHED_BARRIER(st->s); + } + + /* --BEGIN ERROR HANDLING-- */ + /* --END ERROR HANDLING-- */ +fn_fail: + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME sched_cb_gcn_copy_mask +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +static int sched_cb_gcn_copy_mask(MPID_Comm *comm, int tag, void *state) +{ + int mpi_errno = MPI_SUCCESS; + struct gcn_state *st = state; + + MPIU_THREAD_CS_ENTER(CONTEXTID,); + if (st->comm_ptr->context_id < lowestContextId) { + lowestContextId = st->comm_ptr->context_id; + } + if (mask_in_use || (st->comm_ptr->context_id != lowestContextId)) { + memset(st->local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int)); + st->own_mask = 0; + } else { + MPIU_Memcpy(st->local_mask, context_mask, MPIR_MAX_CONTEXT_MASK * sizeof(uint32_t)); + mask_in_use = 1; + st->own_mask = 1; } + MPIU_THREAD_CS_ENTER(CONTEXTID,); - if (st->ctx0) - *st->ctx0 = newctxid; - if (st->ctx1) - *st->ctx1 = newctxid; + mpi_errno = st->comm_ptr->coll_fns->Iallreduce_sched(MPI_IN_PLACE, st->local_mask, MPIR_MAX_CONTEXT_MASK, + MPI_UINT32_T, MPI_BAND, st->comm_ptr, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPID_SCHED_BARRIER(st->s); + + mpi_errno = MPID_Sched_cb(&sched_cb_gcn_allocate_cid, st, st->s); + if (mpi_errno) MPIU_ERR_POP(mpi_errno); + MPID_SCHED_BARRIER(st->s); fn_fail: return mpi_errno; } -/* Does the meat of the algorithm, adds the relevant entries to the schedule. - * Assigns the resulting value to *ctx0 and *ctx1, as long as those respective - * pointers are non-NULL. */ -/* FIXME this version only works for single-threaded code, it will totally fail - * for any multithreaded communicator creation */ +/** The multi-threaded nonblocking algorithm of getting a context id + * + * 1. It shares the same global flag 'mask_in_use' with other communicator + * functions to protect access to context_mask. And use CONTEXTID lock to + * protext critical sections. + * + * 2. It uses the same algorithm as multithreaded MPI_Comm_dup (multi-threaded + * vertion of MPIR_Get_contextid_sparse_group) to allocate a context id, but in + * a nonblocking way. In the case of conflicts, the algorithm needs to retry + * the allocation process again. In the nonblocking algorithm, 1) new entries + * are inserted to the end of schedule to replace the 'while' loop in + * MPI_Comm_dup algorithm; 2) all arguments passed to sched_get_cid_nonblock + * are saved to gcn_state in order to be called in the future; 3) in + * sched_cb_gcn_allocate_cid, if the first try failed, it will insert + * sched_cb_gcn_copy_mask to the schedule again. + * + * 3. There is a subtle difference between INTRACOMM and INTERCOMM when + * duplicating a communicator. They needed to be treated differently in + * current algorithm. Specifically, 1) when calling sched_get_cid_nonblock, the + * parameters are different; 2) updating newcommp->recvcontext_id in + * MPIR_Get_intercomm_contextid_nonblock has been moved to sched_cb_gcn_bcast + * because this should happen after sched_cb_gcn_allocate_cid has succeed. + */ #undef FUNCNAME -#define FUNCNAME gcn_sch +#define FUNCNAME sched_get_cid_nonblock #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -static int gcn_sch(MPID_Comm *comm_ptr, MPIR_Context_id_t *ctx0, MPIR_Context_id_t *ctx1, MPID_Sched_t s) +static int sched_get_cid_nonblock(MPID_Comm *comm_ptr, MPIR_Context_id_t *ctx0, + MPIR_Context_id_t *ctx1, MPID_Sched_t s, MPID_Comm_kind_t gcn_cid_kind) { int mpi_errno = MPI_SUCCESS; struct gcn_state *st = NULL; MPIU_CHKPMEM_DECL(1); - MPIU_Assert(comm_ptr->comm_kind == MPID_INTRACOMM); - - /* first do as much local setup as we can */ + MPIU_THREAD_CS_ENTER(CONTEXTID,); if (initialize_context_mask) { MPIR_Init_contextid(); } @@ -1280,22 +1389,22 @@ static int gcn_sch(MPID_Comm *comm_ptr, MPIR_Context_id_t *ctx0, MPIR_Context_id MPIU_CHKPMEM_MALLOC(st, struct gcn_state *, sizeof(struct gcn_state), mpi_errno, "gcn_state"); st->ctx0 = ctx0; st->ctx1 = ctx1; - MPIU_Memcpy(st->local_mask, context_mask, MPIR_MAX_CONTEXT_MASK * sizeof(uint32_t)); - - mpi_errno = comm_ptr->coll_fns->Iallreduce_sched(MPI_IN_PLACE, st->local_mask, MPIR_MAX_CONTEXT_MASK, - MPI_UINT32_T, MPI_BAND, comm_ptr, s); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - - MPID_SCHED_BARRIER(s); + if (gcn_cid_kind == MPID_INTRACOMM) { + st->comm_ptr = comm_ptr; + st->comm_ptr_inter = NULL; + } else { + st->comm_ptr = comm_ptr->local_comm; + st->comm_ptr_inter = comm_ptr; + } + st->s = s; + st->gcn_cid_kind = gcn_cid_kind; + *(st->ctx0) = 0; + MPIU_THREAD_CS_EXIT(CONTEXTID,); - mpi_errno = MPID_Sched_cb(&gcn_helper, st, s); + mpi_errno = MPID_Sched_cb(&sched_cb_gcn_copy_mask, st, s); if (mpi_errno) MPIU_ERR_POP(mpi_errno); - MPID_SCHED_BARRIER(s); - mpi_errno = MPID_Sched_cb(&MPIR_Sched_cb_free_buf, st, s); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - MPIU_CHKPMEM_COMMIT(); fn_exit: return mpi_errno; @@ -1328,7 +1437,7 @@ int MPIR_Get_contextid_nonblock(MPID_Comm *comm_ptr, MPID_Comm *newcommp, MPID_R if (mpi_errno) MPIU_ERR_POP(mpi_errno); /* add some entries to it */ - mpi_errno = gcn_sch(comm_ptr, &newcommp->context_id, &newcommp->recvcontext_id, s); + mpi_errno = sched_get_cid_nonblock(comm_ptr, &newcommp->context_id, &newcommp->recvcontext_id, s, MPID_INTRACOMM); if (mpi_errno) MPIU_ERR_POP(mpi_errno); /* finally, kick off the schedule and give the caller a request */ @@ -1353,7 +1462,6 @@ int MPIR_Get_intercomm_contextid_nonblock(MPID_Comm *comm_ptr, MPID_Comm *newcom int mpi_errno = MPI_SUCCESS; int tag; MPID_Sched_t s; - MPID_Comm *lcomm = NULL; MPID_MPI_STATE_DECL(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID_NONBLOCK); MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID_NONBLOCK); @@ -1363,7 +1471,6 @@ int MPIR_Get_intercomm_contextid_nonblock(MPID_Comm *comm_ptr, MPID_Comm *newcom mpi_errno = MPIR_Setup_intercomm_localcomm(comm_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } - lcomm = comm_ptr->local_comm; /* now create a schedule */ mpi_errno = MPID_Sched_next_tag(comm_ptr, &tag); @@ -1374,22 +1481,7 @@ int MPIR_Get_intercomm_contextid_nonblock(MPID_Comm *comm_ptr, MPID_Comm *newcom /* add some entries to it */ /* first get a context ID over the local comm */ - mpi_errno = gcn_sch(lcomm, &newcommp->recvcontext_id, NULL, s); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - - MPID_SCHED_BARRIER(s); - - if (comm_ptr->rank == 0) { - newcommp->recvcontext_id = -1; - mpi_errno = MPID_Sched_recv(&newcommp->context_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, 0, comm_ptr, s); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - mpi_errno = MPID_Sched_send(&newcommp->recvcontext_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, 0, comm_ptr, s); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - MPID_SCHED_BARRIER(s); - } - - mpi_errno = lcomm->coll_fns->Ibcast_sched(&newcommp->context_id, 1, - MPIR_CONTEXT_ID_T_DATATYPE, 0, lcomm, s); + mpi_errno = sched_get_cid_nonblock(comm_ptr, &newcommp->recvcontext_id, &newcommp->context_id, s, MPID_INTERCOMM); if (mpi_errno) MPIU_ERR_POP(mpi_errno); /* finally, kick off the schedule and give the caller a request */ diff --git a/test/mpi/.gitignore b/test/mpi/.gitignore index d4b846f..f3a1916 100644 --- a/test/mpi/.gitignore +++ b/test/mpi/.gitignore @@ -600,6 +600,7 @@ /comm/comm_group_rand /comm/comm_idup /comm/comm_idup_mul +/comm/comm_idup_overlap /comm/comm_info /comm/commcreate1 /comm/ctxsplit diff --git a/test/mpi/comm/testlist b/test/mpi/comm/testlist index c5ed8d8..b3987af 100644 --- a/test/mpi/comm/testlist +++ b/test/mpi/comm/testlist @@ -27,8 +27,8 @@ comm_group_rand 8 mpiversion=3.0 comm_idup 2 mpiversion=3.0 comm_idup 4 mpiversion=3.0 comm_idup 9 mpiversion=3.0 -comm_idup_mul 2 mpiversion=3.0 xfail=ticket1935 -comm_idup_overlap 2 mpiversion=3.0 xfail=ticket1935 +comm_idup_mul 2 mpiversion=3.0 +comm_idup_overlap 2 mpiversion=3.0 dup_with_info 2 mpiversion=3.0 dup_with_info 4 mpiversion=3.0 dup_with_info 9 mpiversion=3.0 ----------------------------------------------------------------------- Summary of changes: src/mpi/comm/commutil.c | 202 ++++++++++++++------ test/mpi/.gitignore | 3 + test/mpi/comm/testlist | 4 +- test/mpi/threads/comm/Makefile.am | 2 +- .../comm/{comm_dup_deadlock.c => comm_idup.c} | 16 +- test/mpi/threads/comm/{ctxdup.c => ctxidup.c} | 38 ++-- test/mpi/threads/comm/testlist | 2 + 7 files changed, 184 insertions(+), 83 deletions(-) copy test/mpi/threads/comm/{comm_dup_deadlock.c => comm_idup.c} (79%) copy test/mpi/threads/comm/{ctxdup.c => ctxidup.c} (77%) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-286-g6063f89
by noreply＠mpich.org 30 May '14

30 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 6063f89812b3b1b718abffa1b0c895cd5a19754c (commit) from 725ffc91efbfc7414be7de9b26f5f01bdd6a7cce (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/6063f89812b3b1b718abffa1b0c895cd5… commit 6063f89812b3b1b718abffa1b0c895cd5a19754c Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Fri May 30 15:00:15 2014 -0500 pamid: fix request completion race condition In a multi-threaded environment where one thread is waiting for a request to complete, the instant the request completion counter is set to zero by a different thread the waiting thread will begin to free the request resources. The completing thread must not touch any memory from the old request object after setting the completion count to zero. See ticket #2103 diff --git a/src/mpid/pamid/src/onesided/mpid_1s.c b/src/mpid/pamid/src/onesided/mpid_1s.c index 447530b..fa91cd9 100644 --- a/src/mpid/pamid/src/onesided/mpid_1s.c +++ b/src/mpid/pamid/src/onesided/mpid_1s.c @@ -55,8 +55,7 @@ MPIDI_Win_DoneCB(pami_context_t context, ((req->type >= MPIDI_WIN_REQUEST_COMPARE_AND_SWAP) && (req->origin.completed == req->origin.dt.num_contig))) { - if(req->req_handle) - MPID_cc_set(req->req_handle->cc_ptr, 0); + MPID_Request * req_handle = req->req_handle; if (req->buffer_free) { MPIU_Free(req->buffer); @@ -66,8 +65,22 @@ MPIDI_Win_DoneCB(pami_context_t context, MPIDI_Win_datatype_unmap(&req->target.dt); if (req->accum_headers) MPIU_Free(req->accum_headers); + if (!((req->type > MPIDI_WIN_REQUEST_GET_ACCUMULATE) && (req->type <=MPIDI_WIN_REQUEST_RGET_ACCUMULATE))) MPIU_Free(req); + + if(req_handle) { + + /* The instant this completion counter is set to zero another thread + * may notice the change and begin freeing request resources. The + * thread executing the code in this function must not touch any + * portion of the request structure after decrementing the completion + * counter. + * + * See MPID_Request_release_inline() + */ + MPID_cc_set(req_handle->cc_ptr, 0); + } } MPIDI_Progress_signal(); } diff --git a/src/mpid/pamid/src/onesided/mpid_win_get.c b/src/mpid/pamid/src/onesided/mpid_win_get.c index e3d9885..019d513 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get.c @@ -260,17 +260,27 @@ MPID_Get(void *origin_addr, /* If the get is a local operation, do it here */ if (target_rank == win->comm_ptr->rank) { - size_t offset = req->offset; + /* The operation is not complete until the local copy is performed */ + mpi_errno = MPIR_Localcopy(win->base + req->offset, + target_count, + target_datatype, + origin_addr, + origin_count, + origin_datatype); + + /* The instant this completion counter is set to zero another thread + * may notice the change and begin freeing request resources. The + * thread executing the code in this function must not touch any + * portion of the request structure after decrementing the completion + * counter. + * + * See MPID_Request_release_inline() + */ if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); else MPIU_Free(req); - return MPIR_Localcopy(win->base + offset, - target_count, - target_datatype, - origin_addr, - origin_count, - origin_datatype); + return mpi_errno; } req->target.rank = target_rank; diff --git a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c index ca93a13..420c9e2 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c @@ -139,21 +139,34 @@ MPIDI_Win_GetAccDoneCB(pami_context_t context, if (req->origin.completed == req->target.dt.num_contig + 1) { - if(req->req_handle) - MPID_cc_set(req->req_handle->cc_ptr, 0); + MPID_Request * req_handle = req->req_handle; if (req->buffer_free) { MPIU_Free(req->buffer); req->buffer_free = 0; } if (req->accum_headers) - MPIU_Free(req->accum_headers); + MPIU_Free(req->accum_headers); MPIDI_Win_datatype_unmap(&req->target.dt); MPIDI_Win_datatype_unmap(&req->result.dt); - if( req->type != MPIDI_WIN_REQUEST_RGET_ACCUMULATE ) - MPIU_Free(req); + if( req->type != MPIDI_WIN_REQUEST_RGET_ACCUMULATE ) { + if (req_handle) { + req_handle->mpid.win_req = NULL; + } + MPIU_Free(req); + } + /* The instant this completion counter is set to zero another thread + * may notice the change and begin freeing request resources. The + * thread executing the code in this function must not touch any + * portion of the request structure after decrementing the completion + * counter. + * + * See MPID_Request_release_inline() + */ + if(req_handle) + MPID_cc_set(req_handle->cc_ptr, 0); } MPIDI_Progress_signal(); } diff --git a/src/mpid/pamid/src/onesided/mpid_win_put.c b/src/mpid/pamid/src/onesided/mpid_win_put.c index a552fa0..db5f20c 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_put.c +++ b/src/mpid/pamid/src/onesided/mpid_win_put.c @@ -265,17 +265,27 @@ MPID_Put(const void *origin_addr, /* If the get is a local operation, do it here */ if (target_rank == win->comm_ptr->rank) { - size_t offset = req->offset; + /* The operation is not complete until the local copy is performed */ + mpi_errno = MPIR_Localcopy(origin_addr, + origin_count, + origin_datatype, + win->base + req->offset, + target_count, + target_datatype); + + /* The instant this completion counter is set to zero another thread + * may notice the change and begin freeing request resources. The + * thread executing the code in this function must not touch any + * portion of the request structure after decrementing the completion + * counter. + * + * See MPID_Request_release_inline() + */ if(req->req_handle) MPID_cc_set(req->req_handle->cc_ptr, 0); else MPIU_Free(req); - return MPIR_Localcopy(origin_addr, - origin_count, - origin_datatype, - win->base + offset, - target_count, - target_datatype); + return mpi_errno; } req->target.rank = target_rank; ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/src/onesided/mpid_1s.c | 17 ++++++++++++- src/mpid/pamid/src/onesided/mpid_win_get.c | 24 ++++++++++++++----- .../pamid/src/onesided/mpid_win_get_accumulate.c | 23 +++++++++++++++---- src/mpid/pamid/src/onesided/mpid_win_put.c | 24 ++++++++++++++----- 4 files changed, 67 insertions(+), 21 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-285-g725ffc9
by noreply＠mpich.org 29 May '14

29 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 725ffc91efbfc7414be7de9b26f5f01bdd6a7cce (commit) via a55f4f42b385aa679eb9970061b2e6bac3974cf1 (commit) from 4058ce40d15ef5ce8ba2ce3b90ec0ae3e0568af9 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/725ffc91efbfc7414be7de9b26f5f01bd… commit 725ffc91efbfc7414be7de9b26f5f01bdd6a7cce Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Wed May 28 13:13:19 2014 -0500 pamid: fix win_shared_query diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 73cbb3a..de191b1 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -441,6 +441,7 @@ typedef struct MPIDI_Win_info uint32_t disp_unit; /**< Node's exposure window displacement units */ pami_memregion_t memregion; /**< Memory region descriptor for each node */ uint32_t memregion_used; + MPI_Aint base_size; /**< Node's exposure window base size in bytes */ } MPIDI_Win_info; typedef pthread_mutex_t MPIDI_SHM_MUTEX; diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index be8e2e5..bdb241f 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -354,6 +354,7 @@ MPID_getSharedSegment(MPI_Aint size, win->mpid.info[rank].base_addr = NULL; } win->base = win->mpid.info[rank].base_addr; + win->mpid.info[rank].base_size = size; /* set mutex_lock address and initialize it */ win->mpid.shm->mutex_lock = (pthread_mutex_t *) win->mpid.shm->base_addr; @@ -378,6 +379,7 @@ MPID_getSharedSegment(MPI_Aint size, /* calculate total number of bytes needed */ for (i = 0; i < comm_size; ++i) { + win->mpid.info[i].base_size = tmp_buf[i]; len = tmp_buf[i]; if (*noncontig) /* Round up to next page size */ @@ -508,20 +510,19 @@ MPID_Win_allocate_shared(MPI_Aint size, if (mpi_errno != MPI_SUCCESS) return mpi_errno; - win->mpid.info[0].base_addr = win->base; if (comm_size > 1) { - char *cur_base = (*win_ptr)->base; - for (i = 1; i < comm_size; ++i) { - if (size) { + char *cur_base = (*win_ptr)->base; + for (i = 0; i < comm_size; ++i) { + if (win->mpid.info[i].base_size) { if (noncontig) /* Round up to next page size */ - win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size,pageSize)); - else - win->mpid.info[i].base_addr = (void *) ((MPI_Aint) cur_base + (MPI_Aint) size); - cur_base = win->mpid.info[i].base_addr; - } else { - win->mpid.info[i].base_addr = NULL; - } + win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size,pageSize)); + else + win->mpid.info[i].base_addr = (void *) ((MPI_Aint) cur_base + (MPI_Aint) size); + cur_base = win->mpid.info[i].base_addr; + } else { + win->mpid.info[i].base_addr = NULL; + } } } diff --git a/src/mpid/pamid/src/onesided/mpid_win_shared_query.c b/src/mpid/pamid/src/onesided/mpid_win_shared_query.c index 21e70fd..aef5458 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_shared_query.c +++ b/src/mpid/pamid/src/onesided/mpid_win_shared_query.c @@ -39,13 +39,25 @@ int MPID_Win_shared_query(MPID_Win *win, int rank, MPI_Aint *size, int *disp_unit, void *base_ptr) { - int mpi_errno = MPI_SUCCESS; + int i, mpi_errno = MPI_SUCCESS; static char FCNAME[] = "MPID_Win_shared_query"; MPIU_ERR_CHKANDSTMT((win->create_flavor != MPI_WIN_FLAVOR_SHARED), mpi_errno, MPI_ERR_RMA_FLAVOR, return mpi_errno, "**rmaflavor"); - *((void **) base_ptr) = (void *) win->base; - *size = win->size; - *disp_unit = win->disp_unit; + + if (rank == MPI_PROC_NULL) { + for (i=0; i<win->comm_ptr->local_size; ++i) { + if (win->mpid.info[i].base_size != 0) { + *((void **) base_ptr) = win->mpid.info[i].base_addr; + *size = win->mpid.info[i].base_size; + *disp_unit = win->mpid.info[i].disp_unit; + break; + } + } + } else { + *((void **) base_ptr) = win->mpid.info[rank].base_addr; + *size = win->mpid.info[rank].base_size; + *disp_unit = win->mpid.info[rank].disp_unit; + } return mpi_errno; } http://git.mpich.org/mpich.git/commitdiff/a55f4f42b385aa679eb9970061b2e6bac… commit a55f4f42b385aa679eb9970061b2e6bac3974cf1 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Wed May 28 12:08:22 2014 -0500 pamid: fix win_allocate_shared error handling Fixes #2101 diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index c9a93fe..be8e2e5 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -154,16 +154,15 @@ int GetPageSize(void *addr, ulong *pageSize) return 0; } -void * +int MPID_getSharedSegment_mmap(MPID_Win * win) { - void * base_addr; int rank, rc, fd; int mpi_errno = MPI_SUCCESS; int errflag = FALSE; int first = 0; - snprintf (win->mpid.shm->shm_key, 63, "/mpich/comm-%d/win_shared", win->comm_ptr->context_id); + snprintf (win->mpid.shm->shm_key, 63, "/mpich.comm-%d.win_shared", win->comm_ptr->context_id); rc = shm_open (win->mpid.shm->shm_key, O_RDWR | O_CREAT | O_EXCL, 0600); if (0 == rc) { @@ -177,17 +176,13 @@ MPID_getSharedSegment_mmap(MPID_Win * win) rc = ftruncate (fd, win->mpid.shm->segment_len); MPIU_ERR_CHKANDJUMP((rc == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - - base_addr = mmap (NULL, win->mpid.shm->segment_len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (base_addr == NULL || base_addr == MAP_FAILED || base_addr == (void *) -1) { /* error */ - if (0 == rank) shm_unlink (win->mpid.shm->shm_key); - MPIU_ERR_CHKANDJUMP((win->mpid.shm->shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - } + win->mpid.shm->base_addr = mmap (NULL, win->mpid.shm->segment_len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + MPIU_ERR_CHKANDJUMP((win->mpid.shm->base_addr == MAP_FAILED), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); close (fd); /* no longer needed */ /* set mutex_lock address and initialize it */ - win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) base_addr; + win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) win->mpid.shm->base_addr; if (1 == first) { MPIDI_SHM_MUTEX_INIT(win); } @@ -198,14 +193,15 @@ MPID_getSharedSegment_mmap(MPID_Win * win) win->mpid.shm->allocated = 1; fn_exit: - return base_addr; + return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: + shm_unlink (win->mpid.shm->shm_key); goto fn_exit; /* --END ERROR HANDLING-- */ } -void * +int MPID_getSharedSegment_sysv(MPID_Win * win) { int mpi_errno = MPI_SUCCESS; @@ -214,7 +210,6 @@ MPID_getSharedSegment_sysv(MPID_Win * win) int rank; char *cp; int shm_flag = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR; - void * base_addr; shm_key = (uint32_t) -1; @@ -272,11 +267,11 @@ MPID_getSharedSegment_sysv(MPID_Win * win) win->mpid.shm->shm_id = shmget(shm_key, win->mpid.shm->segment_len, shm_flag); MPIU_ERR_CHKANDJUMP((win->mpid.shm->shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - base_addr = (void *) shmat(win->mpid.shm->shm_id,0,0); - MPIU_ERR_CHKANDJUMP((base_addr == (void*) -1), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); + win->mpid.shm->base_addr = (void *) shmat(win->mpid.shm->shm_id,0,0); + MPIU_ERR_CHKANDJUMP((win->mpid.shm->base_addr == (void*) -1), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); /* set mutex_lock address and initialize it */ - win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) base_addr; + win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) win->mpid.shm->base_addr; MPIDI_SHM_MUTEX_INIT(win); /* successfully created shm segment - shared the key with other tasks */ @@ -288,15 +283,15 @@ MPID_getSharedSegment_sysv(MPID_Win * win) win->mpid.shm->shm_id = shmget(shm_key, 0, 0); if (win->mpid.shm->shm_id != -1) { /* shm segment is available */ - base_addr = (void *) shmat(win->mpid.shm->shm_id,0,0); + win->mpid.shm->base_addr = (void *) shmat(win->mpid.shm->shm_id,0,0); } - win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) base_addr; + win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) win->mpid.shm->base_addr; } win->mpid.shm->allocated = 1; fn_exit: - return base_addr; + return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: goto fn_exit; @@ -401,12 +396,13 @@ MPID_getSharedSegment(MPI_Aint size, * data buffer - possibly padded if non-contiguous. */ #ifdef USE_SYSV_SHM - win->mpid.shm->base_addr = MPID_getSharedSegment_sysv(win); + mpi_errno = MPID_getSharedSegment_sysv(win); #elif USE_MMAP_SHM - win->mpid.shm->base_addr = MPID_getSharedSegment_mmap(win); + mpi_errno = MPID_getSharedSegment_mmap(win); #else MPID_Abort(NULL, MPI_ERR_RMA_SHARED, -1, "RMA shared segment error"); #endif + if (mpi_errno) MPIU_ERR_POP(mpi_errno); /* increment the shared counter */ win->mpid.shm->shm_count=(int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(MPIDI_SHM_MUTEX)); @@ -504,7 +500,9 @@ MPID_Win_allocate_shared(MPI_Aint size, mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); comm_size = (*win_ptr)->comm_ptr->local_size; - MPID_getSharedSegment(size, disp_unit,comm_ptr, win_ptr, &pageSize, &noncontig); + mpi_errno = MPID_getSharedSegment(size, disp_unit,comm_ptr, win_ptr, &pageSize, &noncontig); + if (mpi_errno != MPI_SUCCESS) + return mpi_errno; mpi_errno = MPIDI_Win_allgather(size,win_ptr); if (mpi_errno != MPI_SUCCESS) ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_datatypes.h | 1 + .../pamid/src/onesided/mpid_win_allocate_shared.c | 65 ++++++++++---------- .../pamid/src/onesided/mpid_win_shared_query.c | 20 +++++- 3 files changed, 49 insertions(+), 37 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-283-g4058ce4
by noreply＠mpich.org 28 May '14

28 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 4058ce40d15ef5ce8ba2ce3b90ec0ae3e0568af9 (commit) from 33aacdb8fe417c3279b3ee5508e5ed93bdd93f22 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/4058ce40d15ef5ce8ba2ce3b90ec0ae3e… commit 4058ce40d15ef5ce8ba2ce3b90ec0ae3e0568af9 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Wed May 28 13:06:36 2014 -0500 Don't build GPFS hints if headers not found User is responsible for setting CPPFLAGS and LDFLAGS to find headers/libraries needed for gpfs_fcntl() routines. If not found or not available, don't compile that bit of code. Signed-off-by: Paul Coffman <pkcoff(a)us.ibm.com> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 2d890e5..eef476d 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -380,7 +380,8 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count, void gpfs_wr_access_start(int fd, ADIO_Offset offset, ADIO_Offset length) { - int rc; + int rc=0; +#ifdef HAVE_GPFS_FCNTL_H struct { gpfsFcntlHeader_t header; gpfsAccessRange_t access; @@ -397,12 +398,14 @@ void gpfs_wr_access_start(int fd, ADIO_Offset offset, ADIO_Offset length) take_locks.access.isWrite = 1; rc = gpfs_fcntl(fd, &take_locks); +#endif ADIOI_Assert(rc == 0); } void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) { - int rc; + int rc=0; +#ifdef HAVE_GPFS_FCNTL_H struct { gpfsFcntlHeader_t header; gpfsFreeRange_t free; @@ -419,6 +422,7 @@ void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) free_locks.free.length = length; rc = gpfs_fcntl(fd, &free_locks); +#endif ADIOI_Assert(rc == 0); } ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-282-g33aacdb
by noreply＠mpich.org 27 May '14

27 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 33aacdb8fe417c3279b3ee5508e5ed93bdd93f22 (commit) via 2b401aaf6188830db802bd9432028ca3526a699c (commit) via 2ff6d4931312b4e17fe1c813ef6b2450732be88d (commit) via 3018d28c8e320e8432ca45b135ffa98ebce35040 (commit) via c892ac58e4ac7645429030d02cff648dcbd7f6d2 (commit) from 3d3a4b3a1038f7809ffdba5a7ebe1e02a7325daf (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/33aacdb8fe417c3279b3ee5508e5ed93b… commit 33aacdb8fe417c3279b3ee5508e5ed93bdd93f22 Author: Su Huang <suhuang(a)us.ibm.com> Date: Tue May 27 07:58:50 2014 -0400 pamid: mutiple fixes for RMA shared window The following fixes applied to PAMID for MPI_Win_allocate_shared: 1) include fcntl.h in mpid_win_allocate_shared.c 1) for one task window, needs to allocate space for mutex_lock and initialize it. the address of allocated memory is stored in win->mpid.shm->base_addr 2) in MPID_Win_fetch_and_op(), the base address for target_rank is changed from base = win->mpid.shm->base_addr; to base = win->mpid.info[target_rank].base_addr; 3) MPID_Win_free(): to free shared window for one task window, change from MPIU_Free((*win_ptr)->base); to MPIU_Free((*win_ptr)->mpid.shm->base_addr); 4) MPID_Abort() requires arguments. MPID_Abort(NULL, MPI_ERR_RMA_SHARED, -1, "MPI_Win_free error"); Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 161d0d8..c9a93fe 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -23,7 +23,7 @@ #include <sys/shm.h> #include <sys/ipc.h> #include <sys/stat.h> - +#include <fcntl.h> #include <sys/mman.h> #undef FUNCNAME @@ -40,7 +40,6 @@ extern int mpidi_dynamic_tasking; #define ALIGN_BOUNDARY 128 /* Align data structures to cache line */ #define PAD_SIZE(s) (ALIGN_BOUNDARY - (sizeof(s) & (ALIGN_BOUNDARY-1))) - int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { int comm_size, i; int mpi_errno = PAMI_SUCCESS; @@ -318,6 +317,7 @@ MPID_getSharedSegment(MPI_Aint size, MPI_Aint pageSize,pageSize2, len,new_size; MPID_Win *win; int padSize; + void *base_pp; win = *win_ptr; comm_size = win->comm_ptr->local_size; @@ -332,26 +332,39 @@ MPID_getSharedSegment(MPI_Aint size, if (comm_size == 1) { /* Do not use shared memory when there is only one rank on the node */ + /* 'size' must not be < 0 */ + MPIU_ERR_CHKANDSTMT(size < 0 , mpi_errno, MPI_ERR_SIZE,return mpi_errno, "**rmasize"); + + /* The beginning of the heap allocation contains a control block + * before the data begins. + */ + new_size = MPIDI_ROUND_UP_PAGESIZE(sizeof(MPIDI_Win_shm_ctrl_t),pageSize); + if (size > 0) { if (*noncontig) - new_size = MPIDI_ROUND_UP_PAGESIZE(size,pageSize); + new_size += MPIDI_ROUND_UP_PAGESIZE(size,pageSize); else - new_size = size; - - win->base = MPIU_Malloc(new_size); - MPIU_ERR_CHKANDJUMP((win->base == NULL), mpi_errno, MPI_ERR_BUFFER, "**bufnull"); - - } else if (size == 0) { - win->base = NULL; - - } else { - /* 'size' must be >= 0 */ - MPIU_ERR_CHKANDSTMT(size >=0 , mpi_errno, MPI_ERR_SIZE,return mpi_errno, "**rmasize"); + new_size += size; } + base_pp = MPIU_Malloc(new_size); + MPID_assert(base_pp !=NULL); if (mpi_errno) MPIU_ERR_POP(mpi_errno); win->mpid.shm->segment_len = new_size; + win->mpid.shm->base_addr = base_pp; + if (size !=0) { + win->mpid.info[rank].base_addr = (void *)((MPI_Aint) base_pp + MPIDI_ROUND_UP_PAGESIZE(sizeof(MPIDI_Win_shm_ctrl_t),pageSize)); + } else { + win->mpid.info[rank].base_addr = NULL; + } + win->base = win->mpid.info[rank].base_addr; + + /* set mutex_lock address and initialize it */ + win->mpid.shm->mutex_lock = (pthread_mutex_t *) win->mpid.shm->base_addr; + win->mpid.shm->shm_count = (int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(pthread_mutex_t)); + MPIDI_SHM_MUTEX_INIT(win); + OPA_fetch_and_add_int((OPA_int_t *) win->mpid.shm->shm_count,1); } else { /* allocate a temporary buffer to gather the 'size' of each buffer on @@ -392,7 +405,7 @@ MPID_getSharedSegment(MPI_Aint size, #elif USE_MMAP_SHM win->mpid.shm->base_addr = MPID_getSharedSegment_mmap(win); #else - MPID_Abort(); + MPID_Abort(NULL, MPI_ERR_RMA_SHARED, -1, "RMA shared segment error"); #endif /* increment the shared counter */ diff --git a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c index 4a6e831..b1a415c 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c +++ b/src/mpid/pamid/src/onesided/mpid_win_fetch_and_op.c @@ -300,7 +300,7 @@ int MPID_Fetch_and_op(const void *origin_addr, void *result_addr, if (win->create_flavor == MPI_WIN_FLAVOR_SHARED) { MPIDI_SHM_MUTEX_LOCK(win); shm_locked = 1; - base = win->mpid.shm->base_addr; + base = win->mpid.info[target_rank].base_addr; disp_unit = win->disp_unit; } diff --git a/src/mpid/pamid/src/onesided/mpid_win_free.c b/src/mpid/pamid/src/onesided/mpid_win_free.c index c31ecf0..e926448 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_free.c +++ b/src/mpid/pamid/src/onesided/mpid_win_free.c @@ -49,10 +49,10 @@ int MPIDI_SHM_Win_free(MPID_Win **win_ptr) munmap ((*win_ptr)->mpid.shm->base_addr, (*win_ptr)->mpid.shm->segment_len); if (0 == (*win_ptr)->comm_ptr->rank) shm_unlink ((*win_ptr)->mpid.shm->shm_key); #else - MPID_Abort(); + MPID_Abort(NULL, MPI_ERR_RMA_SHARED, -1, "MPI_Win_free error"); #endif } else {/* one task on a node */ - MPIU_Free((*win_ptr)->base); + MPIU_Free((*win_ptr)->mpid.shm->base_addr); } MPIU_Free((*win_ptr)->mpid.shm); (*win_ptr)->mpid.shm = NULL; diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 09af8b7..214926a 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -37,14 +37,13 @@ pami_rmw_t zero_rmw_parms; #define MPIDI_QUICKSLEEP usleep(1); #define MAX_NUM_CTRLSEND 1024 /* no more than 1024 outstanding control sends */ - #define MPIDI_SHM_MUTEX_LOCK(win) \ do { \ pthread_mutex_t *shm_mutex = win->mpid.shm->mutex_lock; \ int rval = pthread_mutex_lock(shm_mutex); \ MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_lock", \ "**pthread_lock %s", strerror(rval)); \ - } while (0) + } while (0); #define MPIDI_SHM_MUTEX_UNLOCK(win) \ do { \ @@ -52,7 +51,7 @@ pami_rmw_t zero_rmw_parms; int rval = pthread_mutex_unlock(shm_mutex); \ MPIU_ERR_CHKANDJUMP1(rval, mpi_errno, MPI_ERR_OTHER, "**pthread_unlock", \ "**pthread_unlock %s", strerror(rval)); \ - } while (0) + } while (0); #define MPIDI_SHM_MUTEX_INIT(win) \ do { \ http://git.mpich.org/mpich.git/commitdiff/2b401aaf6188830db802bd9432028ca35… commit 2b401aaf6188830db802bd9432028ca3526a699c Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Fri May 16 15:20:37 2014 -0500 pamid: add a posix (mmap) shared memory impl. for bgq diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 26da181..73cbb3a 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -457,7 +457,11 @@ typedef struct MPIDI_Win_shm_t region associated with it */ void *base_addr; /* base address of shared memory region */ MPI_Aint segment_len; /* size of shared memory region */ - uint32_t shm_id; /* shared memory id */ + union + { + uint32_t shm_id; /* shared memory id - sysv */ + char shm_key[64]; /* shared memory key - posix */ + }; int *shm_count; MPIDI_SHM_MUTEX *mutex_lock; /* shared memory windows -- lock for */ /* accumulate/atomic operations */ diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 10fba99..161d0d8 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -24,6 +24,8 @@ #include <sys/ipc.h> #include <sys/stat.h> +#include <sys/mman.h> + #undef FUNCNAME #define FUNCNAME MPID_Win_allocate_shared #undef FCNAME @@ -154,6 +156,57 @@ int GetPageSize(void *addr, ulong *pageSize) } void * +MPID_getSharedSegment_mmap(MPID_Win * win) +{ + void * base_addr; + int rank, rc, fd; + int mpi_errno = MPI_SUCCESS; + int errflag = FALSE; + int first = 0; + + snprintf (win->mpid.shm->shm_key, 63, "/mpich/comm-%d/win_shared", win->comm_ptr->context_id); + rc = shm_open (win->mpid.shm->shm_key, O_RDWR | O_CREAT | O_EXCL, 0600); + if (0 == rc) + { + first = 1; + } else { + rc = shm_open (win->mpid.shm->shm_key, O_RDWR, 0); + MPIU_ERR_CHKANDJUMP((rc == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + } + + fd = rc; + rc = ftruncate (fd, win->mpid.shm->segment_len); + MPIU_ERR_CHKANDJUMP((rc == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + + + base_addr = mmap (NULL, win->mpid.shm->segment_len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (base_addr == NULL || base_addr == MAP_FAILED || base_addr == (void *) -1) { /* error */ + if (0 == rank) shm_unlink (win->mpid.shm->shm_key); + MPIU_ERR_CHKANDJUMP((win->mpid.shm->shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + } + + close (fd); /* no longer needed */ + + /* set mutex_lock address and initialize it */ + win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) base_addr; + if (1 == first) { + MPIDI_SHM_MUTEX_INIT(win); + } + + mpi_errno = MPIR_Barrier_impl(win->comm_ptr, &errflag); + MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); + + win->mpid.shm->allocated = 1; + +fn_exit: + return base_addr; + /* --BEGIN ERROR HANDLING-- */ +fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + +void * MPID_getSharedSegment_sysv(MPID_Win * win) { int mpi_errno = MPI_SUCCESS; @@ -335,8 +388,9 @@ MPID_getSharedSegment(MPI_Aint size, * data buffer - possibly padded if non-contiguous. */ #ifdef USE_SYSV_SHM - win->mpid.shm->base_addr = - MPID_getSharedSegment_sysv(win); + win->mpid.shm->base_addr = MPID_getSharedSegment_sysv(win); +#elif USE_MMAP_SHM + win->mpid.shm->base_addr = MPID_getSharedSegment_mmap(win); #else MPID_Abort(); #endif @@ -435,40 +489,6 @@ MPID_Win_allocate_shared(MPI_Aint size, win->mpid.info[rank].win = win; win->mpid.info[rank].disp_unit = disp_unit; -#ifdef __BGQ__ - /* verify BG_MAPCOMMONHEAP=1 env. variable is set */ - if (rank == 0) { - assert(NULL!=getenv("BG_MAPCOMMONHEAP")); - baseP = MPIU_Malloc(size+sizeof(pthread_mutex_t)); -#ifdef MPIDI_NO_ASSERT - MPIU_ERR_CHKANDJUMP((baseP == NULL), mpi_errno, MPI_ERR_BUFFER, "**bufnull"); -#else - MPID_assert(baseP != NULL); -#endif - - pthread_mutex_t *mutex = (pthread_mutex_t *)(((uintptr_t) baseP) + size); - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutex_init(mutex, &attr); - } - - int errflag = 0; - mpi_errno = MPIR_Bcast_impl(&baseP, sizeof(char*), MPI_BYTE, 0, - win->comm_ptr, &errflag); - - win->mpid.shm->mutex_lock = (pthread_mutex_t *)(((uintptr_t) baseP) + size); - win->mpid.shm->allocated = 1; - win->mpid.shm->base_addr = baseP; - - win->base = baseP; - win->mpid.info[rank].base_addr = baseP; - - mpi_errno = MPIDI_Win_allgather(size,win_ptr); - if (mpi_errno != MPI_SUCCESS) { - MPIU_Free(win->mpid.shm); - return mpi_errno; - } -#else mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); comm_size = (*win_ptr)->comm_ptr->local_size; MPID_getSharedSegment(size, disp_unit,comm_ptr, win_ptr, &pageSize, &noncontig); @@ -476,6 +496,7 @@ MPID_Win_allocate_shared(MPI_Aint size, mpi_errno = MPIDI_Win_allgather(size,win_ptr); if (mpi_errno != MPI_SUCCESS) return mpi_errno; + win->mpid.info[0].base_addr = win->base; if (comm_size > 1) { char *cur_base = (*win_ptr)->base; @@ -492,7 +513,6 @@ MPID_Win_allocate_shared(MPI_Aint size, } } } -#endif *(void**) base_ptr = (void *) win->mpid.info[rank].base_addr; diff --git a/src/mpid/pamid/src/onesided/mpid_win_free.c b/src/mpid/pamid/src/onesided/mpid_win_free.c index b09fe6c..c31ecf0 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_free.c +++ b/src/mpid/pamid/src/onesided/mpid_win_free.c @@ -31,15 +31,6 @@ int MPIDI_SHM_Win_free(MPID_Win **win_ptr) int rc; int mpi_errno = MPI_SUCCESS; -#ifdef __BGQ__ - if ((*win_ptr)->comm_ptr->rank == 0) { - MPIDI_SHM_MUTEX_DESTROY(*win_ptr); - MPIU_Free ((*win_ptr)->base); - } - (*win_ptr)->mpid.shm = NULL; -#endif - -#ifdef __PE__ /* Free shared memory region */ /* free shm_base_addrs that's only used for shared memory windows */ if ((*win_ptr)->mpid.shm->allocated) { @@ -48,17 +39,23 @@ int MPIDI_SHM_Win_free(MPID_Win **win_ptr) if ((*win_ptr)->comm_ptr->rank == 0) { MPIDI_SHM_MUTEX_DESTROY(*win_ptr); } +#ifdef USE_SYSV_SHM mpi_errno = shmdt((*win_ptr)->mpid.shm->base_addr); if ((*win_ptr)->comm_ptr->rank == 0) { rc=shmctl((*win_ptr)->mpid.shm->shm_id,IPC_RMID,NULL); MPIU_ERR_CHKANDJUMP((rc == -1), errno,MPI_ERR_RMA_SHARED, "**shmctl"); } +#elif USE_MMAP_SHM + munmap ((*win_ptr)->mpid.shm->base_addr, (*win_ptr)->mpid.shm->segment_len); + if (0 == (*win_ptr)->comm_ptr->rank) shm_unlink ((*win_ptr)->mpid.shm->shm_key); +#else + MPID_Abort(); +#endif } else {/* one task on a node */ MPIU_Free((*win_ptr)->base); } MPIU_Free((*win_ptr)->mpid.shm); (*win_ptr)->mpid.shm = NULL; -#endif fn_fail: return mpi_errno; diff --git a/src/mpid/pamid/subconfigure.m4 b/src/mpid/pamid/subconfigure.m4 index 8b196fa..c1992b1 100644 --- a/src/mpid/pamid/subconfigure.m4 +++ b/src/mpid/pamid/subconfigure.m4 @@ -56,6 +56,7 @@ if test "${pamid_platform}" = "PE" ; then with_shared_memory=sysv PM_REQUIRES_PMI=pmi2/poe elif test "${pamid_platform}" = "BGQ" ; then + with_shared_memory=mmap MPID_DEFAULT_CROSS_FILE=${master_top_srcdir}/src/mpid/pamid/cross/bgq8 MPID_DEFAULT_PM=no fi http://git.mpich.org/mpich.git/commitdiff/2ff6d4931312b4e17fe1c813ef6b24507… commit 2ff6d4931312b4e17fe1c813ef6b2450732be88d Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Fri May 16 15:26:38 2014 -0500 pamid: move pe-specific code to sysv function diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index b773ce7..26da181 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -36,6 +36,8 @@ #include "pami.h" #include "mpidi_trace.h" +#include "opa_primitives.h" + #if (MPIU_HANDLE_ALLOCATION_METHOD == MPIU_HANDLE_ALLOCATION_THREAD_LOCAL) && defined(__BGQ__) struct MPID_Request; typedef struct @@ -443,6 +445,12 @@ typedef struct MPIDI_Win_info typedef pthread_mutex_t MPIDI_SHM_MUTEX; +typedef struct MPIDI_Win_shm_ctrl_t +{ + MPIDI_SHM_MUTEX mutex_lock; + OPA_int_t active; +} MPIDI_Win_shm_ctrl_t; + typedef struct MPIDI_Win_shm_t { int allocated; /* flag: TRUE iff this window has a shared memory diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 1ecd9b5..10fba99 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -153,168 +153,206 @@ int GetPageSize(void *addr, ulong *pageSize) return 0; } +void * +MPID_getSharedSegment_sysv(MPID_Win * win) +{ + int mpi_errno = MPI_SUCCESS; + int errflag = FALSE; + uint32_t shm_key; + int rank; + char *cp; + int shm_flag = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR; + void * base_addr; + + shm_key = (uint32_t) -1; + + rank = win->comm_ptr->rank; + + if (rank == 0) { +#ifdef DYNAMIC_TASKING + /* generate an appropriate key */ + if (!mpidi_dynamic_tasking) { + cp = getenv("MP_I_PMD_PID"); + if (cp) { + shm_key = atoi(cp); + shm_key = shm_key & 0x07ffffff; + shm_key = shm_key | 0x80000000; + } else { + cp = getenv("MP_PARTITION"); + if (cp ) { + shm_key = atol(cp); + shm_key = (shm_key << 16) + SHM_KEY_TAIL; + } else { + TRACE_ERR("ERROR MP_PARTITION not set \n"); + } + } + } else { + cp = getenv("MP_I_KEY_RANGE"); + if (cp) { + sscanf(cp, "0x%x", &shm_key); + shm_key = shm_key | 0x80; + } else { + TRACE_ERR("ERROR MP_I_KEY_RANGE not set \n"); + } + } +#else + cp = getenv("MP_I_PMD_PID"); + if (cp) { + shm_key = atoi(cp); + shm_key = shm_key & 0x07ffffff; + shm_key = shm_key | 0x80000000; + } else { + cp = getenv("MP_PARTITION"); + if (cp ) { + shm_key = atol(cp); + shm_key = (shm_key << 16); +#ifdef SHMCC_KEY_TAIL + shm_key += SHMCC_KEY_TAIL; +#endif + } else { + TRACE_ERR("ERROR MP_PARTITION not set \n"); + } + } +#endif + + MPID_assert(shm_key != -1); + + win->mpid.shm->shm_id = shmget(shm_key, win->mpid.shm->segment_len, shm_flag); + MPIU_ERR_CHKANDJUMP((win->mpid.shm->shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); + + base_addr = (void *) shmat(win->mpid.shm->shm_id,0,0); + MPIU_ERR_CHKANDJUMP((base_addr == (void*) -1), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); + + /* set mutex_lock address and initialize it */ + win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) base_addr; + MPIDI_SHM_MUTEX_INIT(win); + + /* successfully created shm segment - shared the key with other tasks */ + mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, win->comm_ptr, &errflag); + + } else { /* task other than task 0 */ + mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, win->comm_ptr, &errflag); + MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); + + win->mpid.shm->shm_id = shmget(shm_key, 0, 0); + if (win->mpid.shm->shm_id != -1) { /* shm segment is available */ + base_addr = (void *) shmat(win->mpid.shm->shm_id,0,0); + } + win->mpid.shm->mutex_lock = (MPIDI_SHM_MUTEX *) base_addr; + } + + win->mpid.shm->allocated = 1; + +fn_exit: + return base_addr; + /* --BEGIN ERROR HANDLING-- */ +fn_fail: + goto fn_exit; + /* --END ERROR HANDLING-- */ +} + int -MPID_getSharedSegment(MPI_Aint size, - int disp_unit, - MPID_Comm * comm_ptr, - void **base_ptr, - MPID_Win **win_ptr, - MPI_Aint *pSize, - int *noncontig) +MPID_getSharedSegment(MPI_Aint size, + int disp_unit, + MPID_Comm * comm_ptr, + MPID_Win **win_ptr, + MPI_Aint * pSize, + int * noncontig) { int mpi_errno = MPI_SUCCESS; - void **base_pp = base_ptr; int i, comm_size, rank; - uint32_t shm_key; - int shm_id; - MPI_Aint *tmp_buf; int errflag = FALSE; MPI_Aint pageSize,pageSize2, len,new_size; - char *cp; MPID_Win *win; int padSize; - int shm_flag = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR; win = *win_ptr; comm_size = win->comm_ptr->local_size; rank = win->comm_ptr->rank; - tmp_buf = MPIU_Malloc( 2*comm_size*sizeof(MPI_Aint)); if (mpi_errno) MPIU_ERR_POP(mpi_errno); GetPageSize((void *) win_ptr, (ulong *) &pageSize); *pSize = pageSize; win->mpid.shm->segment_len = 0; + if (comm_size == 1) { - if (size > 0) { - if (*noncontig) - new_size = MPIDI_ROUND_UP_PAGESIZE(size,pageSize); - else - new_size = size; - *base_pp = MPIU_Malloc(new_size); - #ifndef MPIDI_NO_ASSERT - MPID_assert(*base_pp != NULL); - #else - MPIU_ERR_CHKANDJUMP((*base_pp == NULL), mpi_errno, MPI_ERR_BUFFER, "**bufnull"); - #endif - } else if (size == 0) { - *base_pp = NULL; - } else { - MPIU_ERR_CHKANDSTMT(size >=0 , mpi_errno, MPI_ERR_SIZE,return mpi_errno, "**rmasize"); - } + /* Do not use shared memory when there is only one rank on the node */ + + if (size > 0) { + if (*noncontig) + new_size = MPIDI_ROUND_UP_PAGESIZE(size,pageSize); + else + new_size = size; + + win->base = MPIU_Malloc(new_size); + MPIU_ERR_CHKANDJUMP((win->base == NULL), mpi_errno, MPI_ERR_BUFFER, "**bufnull"); + + } else if (size == 0) { + win->base = NULL; + + } else { + /* 'size' must be >= 0 */ + MPIU_ERR_CHKANDSTMT(size >=0 , mpi_errno, MPI_ERR_SIZE,return mpi_errno, "**rmasize"); + } + if (mpi_errno) MPIU_ERR_POP(mpi_errno); - win->mpid.shm->segment_len = new_size; - win->mpid.info[rank].base_addr = *base_pp; - win->base = *base_pp; - } else { - tmp_buf[rank] = (MPI_Aint) size; - mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, - tmp_buf, 1 * sizeof(MPI_Aint), MPI_BYTE, - (*win_ptr)->comm_ptr, &errflag); - if (mpi_errno) MPIU_ERR_POP(mpi_errno); - - /* calculate total number of bytes needed */ - for (i = 0; i < comm_size; ++i) { - len = tmp_buf[i]; - if (*noncontig) + + win->mpid.shm->segment_len = new_size; + + } else { + /* allocate a temporary buffer to gather the 'size' of each buffer on + * the node to determine the amount of shared memory to allocate + */ + MPI_Aint *tmp_buf; + tmp_buf = MPIU_Malloc (2*comm_size*sizeof(MPI_Aint)); + tmp_buf[rank] = (MPI_Aint) size; + mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, + tmp_buf, 1 * sizeof(MPI_Aint), MPI_BYTE, + (*win_ptr)->comm_ptr, &errflag); + if (mpi_errno) { + MPIU_Free(tmp_buf); + MPIU_ERR_POP(mpi_errno); + } + + /* calculate total number of bytes needed */ + for (i = 0; i < comm_size; ++i) { + len = tmp_buf[i]; + if (*noncontig) /* Round up to next page size */ - win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(len,pageSize); - else - win->mpid.shm->segment_len += len; - } - len = len + 128; /* needed for mutex_lock etc */ - /* get shared segment */ - - shm_key=-1; - if (rank == 0) { - #ifdef DYNAMIC_TASKING - /* generate an appropriate key */ - if (!mpidi_dynamic_tasking) { - cp = getenv("MP_I_PMD_PID"); - if (cp) { - shm_key = atoi(cp); - shm_key = shm_key & 0x07ffffff; - shm_key = shm_key | 0x80000000; - } else { - cp = getenv("MP_PARTITION"); - if (cp ) { - shm_key = atol(cp); - shm_key = (shm_key << 16) + SHM_KEY_TAIL; - } else { - TRACE_ERR("ERROR MP_PARTITION not set \n"); - } - } - } else { - cp = getenv("MP_I_KEY_RANGE"); - if (cp) { - sscanf(cp, "0x%x", &shm_key); - shm_key = shm_key | 0x80; - } else { - TRACE_ERR("ERROR MP_I_KEY_RANGE not set \n"); - } - } - #else - cp = getenv("MP_I_PMD_PID"); - if (cp) { - shm_key = atoi(cp); - shm_key = shm_key & 0x07ffffff; - shm_key = shm_key | 0x80000000; - } else { - cp = getenv("MP_PARTITION"); - if (cp ) { - shm_key = atol(cp); -#ifdef __PE__ - shm_key = (shm_key << 16) + SHMCC_KEY_TAIL; + win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(len,pageSize); + else + win->mpid.shm->segment_len += len; + } + MPIU_Free(tmp_buf); + + /* The beginning of the shared memory allocation contains a control + * block before the data begins. + */ + win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(sizeof(MPIDI_Win_shm_ctrl_t),pageSize); + + /* Get the shared segment which includes the control block header and + * data buffer - possibly padded if non-contiguous. + */ +#ifdef USE_SYSV_SHM + win->mpid.shm->base_addr = + MPID_getSharedSegment_sysv(win); #else - shm_key = (shm_key << 16); + MPID_Abort(); #endif - } else { - TRACE_ERR("ERROR MP_PARTITION not set \n"); - } - } - #endif - MPID_assert(shm_key != -1); - shm_id = shmget(shm_key, win->mpid.shm->segment_len, shm_flag); - MPIU_ERR_CHKANDJUMP((shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - win->mpid.shm->base_addr = (void *) shmat(shm_id,0,0); - MPIU_ERR_CHKANDJUMP((win->mpid.shm->base_addr == NULL), mpi_errno,MPI_ERR_BUFFER, "**bufnull"); - GetPageSize((void *) win->mpid.shm->base_addr, (ulong*)&pageSize2); - MPID_assert(pageSize == pageSize2); - /* set mutex_lock address and initialize it */ - win->mpid.shm->mutex_lock = (pthread_mutex_t *) win->mpid.shm->base_addr; - win->mpid.shm->shm_count=(int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(pthread_mutex_t)); - MPIDI_SHM_MUTEX_INIT(win); - win->mpid.shm->allocated = 1; - /* successfully created shm segment */ - mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, comm_ptr, &errflag); - } else { /* task other than task 0 */ - mpi_errno = MPIR_Bcast_impl((void *) &shm_key, sizeof(int), MPI_CHAR, 0, comm_ptr, &errflag); - MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail"); - MPID_assert(shm_key != -1); - shm_id = shmget(shm_key, 0, 0); - if (shm_id != -1) { /* shm segment is available */ - win->mpid.shm->base_addr = (void *) shmat(shm_id,0,0); - win->mpid.shm->allocated = 1; - MPIU_ERR_CHKANDJUMP((win->mpid.shm->base_addr == (void *) -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - } else { /* node leader failed, no need to try here */ - MPIU_ERR_CHKANDJUMP((shm_id == -1), mpi_errno, MPI_ERR_RMA_SHARED, "**rmashared"); - } - win->mpid.shm->mutex_lock = (pthread_mutex_t *) win->mpid.shm->base_addr; - win->mpid.shm->shm_count=(int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(pthread_mutex_t)); - } - win->mpid.shm->shm_id = shm_id; - OPA_fetch_and_add_int((OPA_int_t *) win->mpid.shm->shm_count,1); - while(*win->mpid.shm->shm_count != comm_size) MPIDI_QUICKSLEEP; /* wait for all ranks complete shmat */ - /* compute the base addresses of each process within the shared memory segment */ - { - padSize=sizeof(pthread_mutex_t) + sizeof(OPA_int_t); - win->base = (void *) ((long) win->mpid.shm->base_addr + (long ) PAD_SIZE(padSize)); - } - *base_pp = win->base; - } + + /* increment the shared counter */ + win->mpid.shm->shm_count=(int *)((MPI_Aint) win->mpid.shm->mutex_lock + (MPI_Aint) sizeof(MPIDI_SHM_MUTEX)); + OPA_fetch_and_add_int((OPA_int_t *) win->mpid.shm->shm_count,1); + + /* wait for all ranks complete */ + while(*win->mpid.shm->shm_count != comm_size) MPIDI_QUICKSLEEP; + + /* compute the base addresses of each process within the shared memory segment */ + win->base = (void *) ((long) win->mpid.shm->base_addr + (long ) MPIDI_ROUND_UP_PAGESIZE(sizeof(MPIDI_Win_shm_ctrl_t),pageSize)); + } fn_exit: - MPIU_Free(tmp_buf); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: @@ -375,7 +413,6 @@ MPID_Win_allocate_shared(MPI_Aint size, MPID_Win *win = NULL; int rank; - void **baseP = base_ptr; MPIDI_Win_info *winfo; int comm_size,i; int noncontig=FALSE; @@ -434,7 +471,7 @@ MPID_Win_allocate_shared(MPI_Aint size, #else mpi_errno=CheckSpaceType(win_ptr,info,&noncontig); comm_size = (*win_ptr)->comm_ptr->local_size; - MPID_getSharedSegment(size, disp_unit,comm_ptr,baseP, win_ptr,&pageSize,&noncontig); + MPID_getSharedSegment(size, disp_unit,comm_ptr, win_ptr, &pageSize, &noncontig); mpi_errno = MPIDI_Win_allgather(size,win_ptr); if (mpi_errno != MPI_SUCCESS) diff --git a/src/mpid/pamid/subconfigure.m4 b/src/mpid/pamid/subconfigure.m4 index f6128da..8b196fa 100644 --- a/src/mpid/pamid/subconfigure.m4 +++ b/src/mpid/pamid/subconfigure.m4 @@ -53,6 +53,7 @@ dnl Set a value for the maximum processor name. MPID_MAX_PROCESSOR_NAME=128 PM_REQUIRES_PMI=pmi2 if test "${pamid_platform}" = "PE" ; then + with_shared_memory=sysv PM_REQUIRES_PMI=pmi2/poe elif test "${pamid_platform}" = "BGQ" ; then MPID_DEFAULT_CROSS_FILE=${master_top_srcdir}/src/mpid/pamid/cross/bgq8 http://git.mpich.org/mpich.git/commitdiff/3018d28c8e320e8432ca45b135ffa98eb… commit 3018d28c8e320e8432ca45b135ffa98ebce35040 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Fri May 16 15:06:37 2014 -0500 pamid: change macro to not reference a local variable diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 8392ea8..1ecd9b5 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -33,9 +33,8 @@ /* in PMD (mp_pmd.c) as well. */ extern int mpidi_dynamic_tasking; -#define MPIDI_PAGESIZE ((MPI_Aint)pageSize) -#define MPIDI_PAGESIZE_MASK (~(MPIDI_PAGESIZE-1)) -#define MPIDI_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_PAGESIZE_MASK)) & MPIDI_PAGESIZE_MASK) +#define MPIDI_PAGESIZE_MASK(y) (~(((MPI_Aint)y)-1)) +#define MPIDI_ROUND_UP_PAGESIZE(x,y) ((((MPI_Aint)x)+(~MPIDI_PAGESIZE_MASK(y))) & MPIDI_PAGESIZE_MASK(y)) #define ALIGN_BOUNDARY 128 /* Align data structures to cache line */ #define PAD_SIZE(s) (ALIGN_BOUNDARY - (sizeof(s) & (ALIGN_BOUNDARY-1))) @@ -189,7 +188,7 @@ MPID_getSharedSegment(MPI_Aint size, if (comm_size == 1) { if (size > 0) { if (*noncontig) - new_size = MPIDI_ROUND_UP_PAGESIZE(size); + new_size = MPIDI_ROUND_UP_PAGESIZE(size,pageSize); else new_size = size; *base_pp = MPIU_Malloc(new_size); @@ -219,7 +218,7 @@ MPID_getSharedSegment(MPI_Aint size, len = tmp_buf[i]; if (*noncontig) /* Round up to next page size */ - win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(len); + win->mpid.shm->segment_len += MPIDI_ROUND_UP_PAGESIZE(len,pageSize); else win->mpid.shm->segment_len += len; } @@ -447,7 +446,7 @@ MPID_Win_allocate_shared(MPI_Aint size, if (size) { if (noncontig) /* Round up to next page size */ - win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size)); + win->mpid.info[i].base_addr =(void *) ((MPI_Aint) cur_base + (MPI_Aint) MPIDI_ROUND_UP_PAGESIZE(size,pageSize)); else win->mpid.info[i].base_addr = (void *) ((MPI_Aint) cur_base + (MPI_Aint) size); cur_base = win->mpid.info[i].base_addr; http://git.mpich.org/mpich.git/commitdiff/c892ac58e4ac7645429030d02cff648dc… commit c892ac58e4ac7645429030d02cff648dcbd7f6d2 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Fri May 16 14:18:03 2014 -0500 pamid: remove unnecessary bgq specialization. diff --git a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c index 332cf47..8392ea8 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c +++ b/src/mpid/pamid/src/onesided/mpid_win_allocate_shared.c @@ -41,38 +41,23 @@ extern int mpidi_dynamic_tasking; int CheckRankOnNode(MPID_Comm * comm_ptr,int *onNode ) { - int comm_size,i; - int mpi_errno=PAMI_SUCCESS; + int comm_size, i; + int mpi_errno = PAMI_SUCCESS; - comm_size = comm_ptr->local_size; + comm_size = comm_ptr->local_size; - *onNode=1; + *onNode = 1; -#ifdef __PE__ - for (i=0; i< comm_size; i++) { - if (comm_ptr->intranode_table[i] == -1) { - *onNode=0; + for (i = 0; i < comm_size; i++) { + if (comm_ptr->intranode_table[i] == -1) { + *onNode = 0; break; - } - } -#else -#ifdef PAMIX_IS_LOCAL_TASK - for (i=0; i< comm_size; i++) { - if (!PAMIX_Task_is_local(comm_ptr->vcr[i]->taskid)) { - *onNode=0; - break; } - } -#else - if (comm_ptr->intranode_table == NULL) - *onNode = 0; -#endif -#endif - + } if (*onNode== 0) { - MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_CONFLICT, - return mpi_errno, "**rmaconflict"); + MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_CONFLICT, + return mpi_errno, "**rmaconflict"); } return mpi_errno; ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_datatypes.h | 14 +- .../pamid/src/onesided/mpid_win_allocate_shared.c | 462 +++++++++++--------- .../pamid/src/onesided/mpid_win_fetch_and_op.c | 2 +- src/mpid/pamid/src/onesided/mpid_win_free.c | 19 +- src/mpid/pamid/src/onesided/mpidi_onesided.h | 5 +- src/mpid/pamid/subconfigure.m4 | 2 + 6 files changed, 284 insertions(+), 220 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-277-g3d3a4b3
by noreply＠mpich.org 26 May '14

26 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 3d3a4b3a1038f7809ffdba5a7ebe1e02a7325daf (commit) from 71f5e5070c49f5507935016e03953d1eeee73aeb (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/3d3a4b3a1038f7809ffdba5a7ebe1e02a… commit 3d3a4b3a1038f7809ffdba5a7ebe1e02a7325daf Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Mon May 26 18:19:52 2014 -0500 Fix a perl script problem met with perl-5.8.8 To print out something like "$vec[$i]->base_addr", where @vec is an integer array, the old perl is confused by "->" and thinks it is an operator. In fact, we just want to print out "->base_addr" literally. Newer perl (e.g., 5.16.2) is fine with this syntax. Change to "$vec[$i]"."->base_addr" to avoid this problem. No review since F08 binding is experimental now. diff --git a/src/binding/fortran/use_mpi_f08/wrappers_c/buildiface b/src/binding/fortran/use_mpi_f08/wrappers_c/buildiface index 2f5620b..2031629 100755 --- a/src/binding/fortran/use_mpi_f08/wrappers_c/buildiface +++ b/src/binding/fortran/use_mpi_f08/wrappers_c/buildiface @@ -550,7 +550,7 @@ EOT # Temp variable declaration for (my $i = 0; $i < $#vec; $i += 3) { - print CFILE " void *buf$vec[$i] = x$vec[$i]->base_addr;\n"; + print CFILE " void *buf$vec[$i] = x$vec[$i]"."->base_addr;\n"; if ($vec[$i + 1] >= 0) { print CFILE " int count$vec[$i] = x$vec[$i+1];\n"; print CFILE " MPI_Datatype dtype$vec[$i] = x$vec[$i+2];\n"; @@ -572,7 +572,7 @@ EOT # Test if a subarray arg is contiguous. If it is, generate a new datatype for it. for (my $i = 0; $i < $#vec; $i += 3) { if ($vec[$i + 1] >= 0) { - print CFILE " if (x$vec[$i]->rank != 0 && !CFI_is_contiguous(x$vec[$i])) {\n"; + print CFILE " if (x$vec[$i]"."->rank != 0 && !CFI_is_contiguous(x$vec[$i])) {\n"; print CFILE " err = cdesc_create_datatype(x$vec[$i], x$vec[$i+1], x$vec[$i+2], &dtype$vec[$i]);\n"; print CFILE " count$vec[$i] = 1;\n"; print CFILE " }\n\n"; ----------------------------------------------------------------------- Summary of changes: .../fortran/use_mpi_f08/wrappers_c/buildiface | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-276-g71f5e50
by noreply＠mpich.org 23 May '14

23 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 71f5e5070c49f5507935016e03953d1eeee73aeb (commit) via f16bfb543764378ad62a05f99c6fd5481996a1bb (commit) via e053df00a4a22febb28438362f477b20cbef18e1 (commit) via 71a21c87809b7d7074eb59d00f29334099b41607 (commit) via ae5b26577bfcc2cb3034fccc2daba3138a264682 (commit) via 8f6179fa8bdd050a0038ab1034a550a7b7e922a5 (commit) via 3c9992ef6a70f6f977e74fe994c9b7e56d3b00c3 (commit) via 8421f3a9c477188aae522caa23969c4ea2c68ea0 (commit) via 67bf775e9534ef87f1cb8dec536a1f2977379abc (commit) via faad606c90bcd14204021bbd0ab0de01351da779 (commit) from 659053cbe6a35e78ad166d72ce927de7bc69f0d4 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/71f5e5070c49f5507935016e03953d1ee… commit 71f5e5070c49f5507935016e03953d1eeee73aeb Author: Rob Latham <robl(a)mcs.anl.gov> Date: Tue May 20 13:59:54 2014 -0500 Free duplicated communicator in error path If something goes wrong in open, the error path might in some cases (such as inconsistent amode) leave the duplicated communicator hanging around. Thanks, Wei-keng While I was here, I improved the comment about why we cannot use one of the built-in operators for the amode check. Signed-off-by: Wei-keng Liao <wkliao(a)eecs.northwestern.edu> diff --git a/src/mpi/romio/mpi-io/open.c b/src/mpi/romio/mpi-io/open.c index e8cdea3..2d72dae 100644 --- a/src/mpi/romio/mpi-io/open.c +++ b/src/mpi/romio/mpi-io/open.c @@ -48,7 +48,7 @@ int MPI_File_open(MPI_Comm comm, ROMIO_CONST char *filename, int amode, { int error_code = MPI_SUCCESS, file_system, flag, tmp_amode=0, rank; char *tmp; - MPI_Comm dupcomm; + MPI_Comm dupcomm = MPI_COMM_NULL; ADIOI_Fns *fsops; static char myname[] = "MPI_FILE_OPEN"; #ifdef MPI_hpux @@ -106,7 +106,12 @@ int MPI_File_open(MPI_Comm comm, ROMIO_CONST char *filename, int amode, MPIR_MPIOInit(&error_code); if (error_code != MPI_SUCCESS) goto fn_fail; -/* check if amode is the same on all processes */ +/* check if amode is the same on all processes: at first glance, one might try + * to use a built-in operator like MPI_BAND, but we need every mpi process to + * agree the amode was not the same. Consider process A with + * MPI_MODE_CREATE|MPI_MODE_RDWR, and B with MPI_MODE_RDWR: MPI_BAND yields + * MPI_MODE_RDWR. A determines amodes are different, but B proceeds having not + * detected an error */ MPI_Allreduce(&amode, &tmp_amode, 1, MPI_INT, ADIO_same_amode, dupcomm); if (tmp_amode == ADIO_AMODE_NOMATCH) { @@ -149,7 +154,6 @@ int MPI_File_open(MPI_Comm comm, ROMIO_CONST char *filename, int amode, /* --BEGIN ERROR HANDLING-- */ if (error_code != MPI_SUCCESS) { - MPI_Comm_free(&dupcomm); goto fn_fail; } /* --END ERROR HANDLING-- */ @@ -194,6 +198,7 @@ fn_exit: return error_code; fn_fail: /* --BEGIN ERROR HANDLING-- */ + if (dupcomm != MPI_COMM_NULL) MPI_Comm_free(&dupcomm); error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code); goto fn_exit; /* --END ERROR HANDLING-- */ http://git.mpich.org/mpich.git/commitdiff/f16bfb543764378ad62a05f99c6fd5481… commit f16bfb543764378ad62a05f99c6fd5481996a1bb Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Mon May 12 12:43:37 2014 -0500 Blue Gene: one-hop aggregator placement Another experimental approach to selecting and placing aggregators on Blue Gene /Q. This one tries to place aggregators in a ring such that they are one hop away from each other. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c index 43236e0..c993021 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c @@ -39,6 +39,7 @@ int gpfsmpio_pthreadio; int gpfsmpio_p2pcontig; int gpfsmpio_balancecontig; int gpfsmpio_devnullio; +int gpfsmpio_bridgeringagg; double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST+1]; double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; @@ -120,6 +121,13 @@ double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; * - 0 (disabled) or 1 (enabled) * - Default is 0 * + * - GPFSMPIO_BRIDGERINGAGG - Relevant only to BGQ. Aggregator placement + * optimization whch forms a 5-d ring around the bridge node starting at + * GPFSMPIO_BRIDGERINGAGG hops away. Experimental performance results + * suggest best value is 1 and only in conjunction with GPFSMPIO_P2PCONTIG + * and GPFSMPIO_BALANCECONTIG. The number of aggregators selected is still + * GPFSMPIO_NAGG_PSET however the bridge node itself is NOT selected. + * */ void ad_gpfs_get_env_vars() { @@ -164,6 +172,10 @@ void ad_gpfs_get_env_vars() { gpfsmpio_devnullio = 0; x = getenv( "GPFSMPIO_DEVNULLIO" ); if (x) gpfsmpio_devnullio = atoi(x); + + gpfsmpio_bridgeringagg = 0; + x = getenv( "GPFSMPIO_BRIDGERINGAGG" ); + if (x) gpfsmpio_bridgeringagg = atoi(x); } /* report timing breakdown for MPI I/O collective call */ diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h index c5b9c84..16ce7cc 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h @@ -68,6 +68,7 @@ extern int gpfsmpio_pthreadio; extern int gpfsmpio_p2pcontig; extern int gpfsmpio_balancecontig; extern int gpfsmpio_devnullio; +extern int gpfsmpio_bridgeringagg; /* Default is, well, kind of complicated. Blue Gene /L and /P had "psets": one * i/o node and all compute nodes wired to it. On Blue Gene /Q that diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c index 72fcfdf..6cadb8f 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c @@ -16,6 +16,7 @@ // Uncomment this line to turn tracing on for the gpfsmpio_balancecontig aggr selection optimization // #define balancecontigtrace 1 +// #define bridgeringaggtrace 1 #include "adio.h" #include "adio_cb_config_list.h" @@ -183,8 +184,132 @@ ADIOI_BG_compute_agg_ranklist_serial_do (const ADIOI_BG_ConfInfo_t *confInfo, /* BES: This should be done in the init routines probably. */ int i, j; int aggTotal; - int distance, numAggs; int *aggList; + + if (gpfsmpio_bridgeringagg > 0) { + + int numAggs = confInfo->aggRatio * confInfo->ioMinSize /*virtualPsetSize*/; + /* the number of aggregators is (numAggs per bridgenode) */ + if(numAggs == 1) + aggTotal = 1; + else + aggTotal = confInfo->numBridgeRanks * numAggs; + + aggList = (int *)ADIOI_Malloc(aggTotal * sizeof(int)); + if(aggTotal == 1) { /* special case when we only have one bridge node */ + + sortstruct *bridgelist = (sortstruct *)ADIOI_Malloc(confInfo->nProcs * sizeof(sortstruct)); + for(i=0; i < confInfo->nProcs; i++) + { + bridgelist[i].bridge = all_procInfo[i].bridgeRank; + bridgelist[i].rank = i; + TRACE_ERR("bridgelist[%d].bridge: %d .rank: %d\n", i, bridgelist[i].bridge, i); + } + + /* This list contains rank->bridge info. Now, we need to sort this list. */ + qsort(bridgelist, confInfo->nProcs, sizeof(sortstruct), intsort); + + aggList[0] = bridgelist[0].bridge; + ADIOI_Free(bridgelist); + + } + else { // aggTotal > 1 + + ADIOI_BG_ProcInfo_t *allProcInfoAggNodeList = (ADIOI_BG_ProcInfo_t *) ADIOI_Malloc(confInfo->nProcs * sizeof(ADIOI_BG_ProcInfo_t)); + int allProcInfoAggNodeListSize = 0; + int maxManhattanDistanceToBridge = 0; + + // for ppn > 1, assign minumum rank as agg candidate + for (i=0;i<confInfo->nProcs;i++) { + int addProcToAggNodeList = 1; + for (j=0;j<allProcInfoAggNodeListSize;j++) { + if ((allProcInfoAggNodeList[j].torusCoords[0] == all_procInfo[i].torusCoords[0]) && + (allProcInfoAggNodeList[j].torusCoords[1] == all_procInfo[i].torusCoords[1]) && + (allProcInfoAggNodeList[j].torusCoords[2] == all_procInfo[i].torusCoords[2]) && + (allProcInfoAggNodeList[j].torusCoords[3] == all_procInfo[i].torusCoords[3]) && + (allProcInfoAggNodeList[j].torusCoords[4] == all_procInfo[i].torusCoords[4]) && + addProcToAggNodeList) { + // proc is in the node list, replace if this rank is smaller + addProcToAggNodeList = 0; + + if (allProcInfoAggNodeList[j].rank > all_procInfo[i].rank) + allProcInfoAggNodeList[j] = all_procInfo[i]; + } + } // for j + if (addProcToAggNodeList) { + allProcInfoAggNodeList[allProcInfoAggNodeListSize] = all_procInfo[i]; + if (allProcInfoAggNodeList[allProcInfoAggNodeListSize].manhattanDistanceToBridge > maxManhattanDistanceToBridge) + maxManhattanDistanceToBridge = allProcInfoAggNodeList[allProcInfoAggNodeListSize].manhattanDistanceToBridge; + allProcInfoAggNodeListSize++; + } + } // for i + +#ifdef bridgeringaggtrace + fprintf(stderr,"allProcInfoAggNodeListSize is %d aggTotal is %d\n",allProcInfoAggNodeListSize,aggTotal); +#endif + + int *aggNodeBridgeList = (int *) ADIOI_Malloc (allProcInfoAggNodeListSize * sizeof(int)); // list of all bridge ranks + int *aggNodeBridgeListNum = (int *) ADIOI_Malloc (allProcInfoAggNodeListSize * sizeof(int)); + for (i=0;i<allProcInfoAggNodeListSize;i++) { + aggNodeBridgeList[i] = -1; + aggNodeBridgeListNum[i] = 0; + } + + int aggNodeBridgeListSize = 0; + for (i=0;i<allProcInfoAggNodeListSize;i++) { + int foundBridge = 0; + for (j=0;(j<aggNodeBridgeListSize && !foundBridge);j++) { + if (aggNodeBridgeList[j] == allProcInfoAggNodeList[i].bridgeRank) { + foundBridge = 1; + aggNodeBridgeListNum[i]++; + } + } + if (!foundBridge) { + aggNodeBridgeList[aggNodeBridgeListSize] = allProcInfoAggNodeList[i].bridgeRank; + aggNodeBridgeListNum[aggNodeBridgeListSize] = 1; + aggNodeBridgeListSize++; + } + } + + // add aggs based on numAggs per bridge, starting at gpfsmpio_bridgeringagg hops and increasing until numAggs aggs found + int currentAggListSize = 0; + for (i=0;i<aggNodeBridgeListSize;i++) { + int currentBridge = aggNodeBridgeList[i]; + int currentNumHops = gpfsmpio_bridgeringagg; + int numAggsAssignedToThisBridge = 0; + while ((numAggsAssignedToThisBridge < numAggs) && (currentNumHops <= maxManhattanDistanceToBridge)) { + for (j=0;j<allProcInfoAggNodeListSize;j++) { + if (allProcInfoAggNodeList[j].bridgeRank == currentBridge) { + if (allProcInfoAggNodeList[j].manhattanDistanceToBridge == currentNumHops) { + aggList[currentAggListSize] = allProcInfoAggNodeList[j].rank; +#ifdef bridgeringaggtrace + printf("Assigned agg rank %d at torus coords %u %u %u %u %u to bridge %d at torus coords %u %u %u %u %u at a distance of %d hops\n",allProcInfoAggNodeList[j].rank,allProcInfoAggNodeList[j].torusCoords[0],allProcInfoAggNodeList[j].torusCoords[1],allProcInfoAggNodeList[j].torusCoords[2],allProcInfoAggNodeList[j].torusCoords[3],allProcInfoAggNodeList[j].torusCoords[4], currentBridge, all_procInfo[currentBridge].torusCoords[0], all_procInfo[currentBridge].torusCoords[1], all_procInfo[currentBridge].torusCoords[2], all_procInfo[currentBridge].torusCoords[3], all_procInfo[currentBridge].torusCoords[4],currentNumHops); +#endif + currentAggListSize++; + numAggsAssignedToThisBridge++; + if (numAggsAssignedToThisBridge >= numAggs) + break; + } + } + } + currentNumHops++; + } // while + ADIOI_Assert(numAggsAssignedToThisBridge == numAggs); + } // for + + ADIOI_Free(allProcInfoAggNodeList); + ADIOI_Free(aggNodeBridgeList); + ADIOI_Free(aggNodeBridgeListNum); + + } // else aggTotal > 1 + + memcpy(tmp_ranklist, aggList, aggTotal*sizeof(int)); + } // gpfsmpio_bridgeringagg > 0 + + else { // gpfsmpio_bridgeringagg unset - default code + + int distance, numAggs; + /* Aggregators will be midpoints between sorted MPI rank lists of who shares a given * bridge node */ @@ -282,9 +407,11 @@ ADIOI_BG_compute_agg_ranklist_serial_do (const ADIOI_BG_ConfInfo_t *confInfo, ADIOI_Free (bridgelist); - ADIOI_Free (aggList); TRACE_ERR("Leaving ADIOI_BG_compute_agg_ranklist_serial_do\n"); + } + + ADIOI_Free (aggList); return aggTotal; } @@ -371,8 +498,8 @@ ADIOI_BG_compute_agg_ranklist_serial ( ADIO_File fd, for (i=0;i<naggs;i++) bridgelistnum[i] = 0; - /* Each entry in this list corresponds with the bridgelist and will - * contain the lowest bridge agg rank on that ion. */ + /* Each entry in this list corresponds with the bridgelist and will contain the lowest bridge + * agg rank on that ion. */ int *summarybridgeminionaggrank = (int *) ADIOI_Malloc (naggs * sizeof(int)); for (i=0;i<naggs;i++) summarybridgeminionaggrank[i] = -1; @@ -408,9 +535,8 @@ ADIOI_BG_compute_agg_ranklist_serial ( ADIO_File fd, bridgelistnum[summaryranklistbridgeindex]++; } - /* at this point summarybridgeminionaggrank has the agg rank of the bridge - * for entries; now need to make each entry the minimum bridge rank for the - * entire ion. */ + /* at this point summarybridgeminionaggrank has the agg rank of the bridge for entries, + * need to make each entry the minimum bridge rank for the entire ion. */ for (i=0;i<numbridges;i++) { int aggIonId = ionlist[i]; int j; diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c index 7cd0ba2..e3a66a7 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c @@ -13,6 +13,8 @@ */ /* #define TRACE_ON */ +// #define bridgeringaggtrace 1 + #include <stdlib.h> #include "../ad_gpfs.h" #include "ad_bg_pset.h" @@ -79,6 +81,35 @@ static int intsort(const void *p1, const void *p2) return(i1->bridgeCoord - i2->bridgeCoord); } +unsigned torusSize[MPIX_TORUS_MAX_DIMS]; +unsigned dimTorus[MPIX_TORUS_MAX_DIMS]; + +/* This function computes the number of hops between the torus coordinates of the + * aggCoords and bridgeCoords parameters. +*/ +static unsigned procManhattanDistance(unsigned *aggCoords, unsigned *bridgeCoords) { + + unsigned totalDistance = 0; + int i; + for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { + unsigned dimDistance = abs(aggCoords[i] - bridgeCoords[i]); + if (dimDistance > 0) { // could torus make it closer? + if (dimTorus[i]) { + if (aggCoords[i] == torusSize[i]) { // is wrap-around closer + if ((bridgeCoords[i]+1) < dimDistance) // assume will use torus link + dimDistance = bridgeCoords[i]+1; + } + else if (bridgeCoords[i] == torusSize[i]) { // is wrap-around closer + if ((aggCoords[i]+1) < dimDistance) // assume will use torus link + dimDistance = aggCoords[i]+1; + } + } + } + totalDistance += dimDistance; + } + return totalDistance; +} + void ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, @@ -102,14 +133,36 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, proc->rank = rank; proc->coreID = hw.coreID; - proc->ionID = MPIX_IO_node_id (); + + if (gpfsmpio_bridgeringagg > 0) { + for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { + proc->torusCoords[i] = hw.Coords[i]; + } +#ifdef bridgeringaggtrace + if (rank == 0) + fprintf(stderr,"Block dimensions:\n"); +#endif + for (i=0;i<MPIX_TORUS_MAX_DIMS;i++) { + torusSize[i] = hw.Size[i]; + dimTorus[i] = hw.isTorus[i]; +#ifdef bridgeringaggtrace + if (rank == 0) + fprintf(stderr,"Dimension %d has %d elements wrap-around value is %d\n",i,torusSize[i],dimTorus[i]); +#endif + } + } MPI_Comm_size(comm, &commsize); + proc->ionID = MPIX_IO_node_id (); + if(size == 1) { proc->iamBridge = 1; proc->bridgeRank = rank; + if (gpfsmpio_bridgeringagg > 0) { + proc->manhattanDistanceToBridge = 0; + } /* Set up the other parameters */ proc->myIOSize = size; @@ -143,8 +196,32 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, (hw.Coords[1] == pers.Network_Config.cnBridge_B) && (hw.Coords[2] == pers.Network_Config.cnBridge_C) && (hw.Coords[3] == pers.Network_Config.cnBridge_D) && - (hw.Coords[4] == pers.Network_Config.cnBridge_E)) + (hw.Coords[4] == pers.Network_Config.cnBridge_E)) { iambridge = 1; /* I am bridge */ + if (gpfsmpio_bridgeringagg > 0) { + proc->manhattanDistanceToBridge = 0; + } + } + else { // calculate manhattan distance to bridge if gpfsmpio_bridgeringagg is set + if (gpfsmpio_bridgeringagg > 0) { + unsigned aggCoords[MPIX_TORUS_MAX_DIMS],manhattanBridgeCoords[MPIX_TORUS_MAX_DIMS]; + aggCoords[0] = hw.Coords[0]; + manhattanBridgeCoords[0] = pers.Network_Config.cnBridge_A; + aggCoords[1] = hw.Coords[1]; + manhattanBridgeCoords[1] = pers.Network_Config.cnBridge_B; + aggCoords[2] = hw.Coords[2]; + manhattanBridgeCoords[2] = pers.Network_Config.cnBridge_C; + aggCoords[3] = hw.Coords[3]; + manhattanBridgeCoords[3] = pers.Network_Config.cnBridge_D; + aggCoords[4] = hw.Coords[4]; + manhattanBridgeCoords[4] = pers.Network_Config.cnBridge_E; + + proc->manhattanDistanceToBridge= procManhattanDistance(aggCoords, manhattanBridgeCoords); +#ifdef bridgeringaggtrace + fprintf(stderr,"agg coords are %u %u %u %u %u bridge coords are %u %u %u %u %u distance is %u\n",aggCoords[0],aggCoords[1],aggCoords[2],aggCoords[3],aggCoords[4],manhattanBridgeCoords[0],manhattanBridgeCoords[1],manhattanBridgeCoords[2],manhattanBridgeCoords[3],manhattanBridgeCoords[4], proc->manhattanDistanceToBridge); +#endif + } + } TRACE_ERR("Bridge coords(%8.8X): %d %d %d %d %d, %d. iambridge %d\n",bridgeCoords, pers.Network_Config.cnBridge_A,pers.Network_Config.cnBridge_B,pers.Network_Config.cnBridge_C,pers.Network_Config.cnBridge_D,pers.Network_Config.cnBridge_E,0, iambridge); diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h index 51ae4a0..d5f36b1 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h @@ -17,6 +17,10 @@ #ifndef AD_BG_PSET_H_ #define AD_BG_PSET_H_ +#ifdef HAVE_MPIX_H +#include <mpix.h> +#endif + /* Keeps specific information to each process, will be exchanged among processes */ typedef struct { int ioNodeIndex; /* similar to psetNum on BGL/BGP */ @@ -31,6 +35,8 @@ typedef struct { node, i.e. psetsize*/ int iamBridge; /* am *I* the bridge rank? */ int __ipad[2]; + unsigned torusCoords[MPIX_TORUS_MAX_DIMS]; /* torus coordinates of node on which this rank resides */ + unsigned manhattanDistanceToBridge; /* number of hops between this rank and the bridge node */ } ADIOI_BG_ProcInfo_t __attribute__((aligned(16))); /* Keeps general information for the whole communicator, only on process 0 */ http://git.mpich.org/mpich.git/commitdiff/e053df00a4a22febb28438362f477b20c… commit e053df00a4a22febb28438362f477b20cbef18e1 Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Wed May 7 16:17:34 2014 -0500 fix to support gpfs hints and deferred open If deferred open enabled, the processes with no i/o will not have the file open. if they try to call gpfs_fcntl() on their invalid file handle, they will get an error. only processes doing any i/o need to set the hints anyway. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 3d11db9..2d890e5 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -551,17 +551,25 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype if ((st_loc==-1) && (end_loc==-1)) { ntimes = 0; /* this process does no writing. */ } - if (ntimes != 0 && getenv("ROMIO_GPFS_DECLARE_ACCESS")!=NULL) { + if (ntimes > 0) { /* only set the gpfs hint if we have io - ie this rank is + an aggregator -- otherwise will fail for deferred open */ + if (getenv("ROMIO_GPFS_DECLARE_ACCESS")!=NULL) { gpfs_wr_access_start(fd->fd_sys, st_loc, end_loc - st_loc); + } } + ADIO_Offset st_loc_ion, end_loc_ion, needs_gpfs_access_cleanup=0; #ifdef BGQPLATFORM - if (getenv("ROMIO_GPFS_DECLARE_ION_ACCESS")!=NULL) { + if (ntimes > 0) { /* only set the gpfs hint if we have io - ie this rank is + an aggregator -- otherwise will fail for deferred open */ + + if (getenv("ROMIO_GPFS_DECLARE_ION_ACCESS")!=NULL) { if (gpfs_find_access_for_ion(fd, st_loc, end_loc, fd_start, fd_end, &st_loc_ion, &end_loc_ion)) { gpfs_wr_access_start(fd->fd_sys, st_loc_ion, end_loc_ion-st_loc_ion); needs_gpfs_access_cleanup=1; } + } } #endif @@ -851,10 +859,6 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype needs_gpfs_access_cleanup=0; } - if (needs_gpfs_access_cleanup) { - gpfs_wr_access_end(fd->fd_sys, end_loc_ion-st_loc_ion, st_loc_ion); - needs_gpfs_access_cleanup=0; - } unsetenv("LIBIOLOG_EXTRA_INFO"); } http://git.mpich.org/mpich.git/commitdiff/71a21c87809b7d7074eb59d00f2933409… commit 71a21c87809b7d7074eb59d00f29334099b41607 Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Tue May 20 11:06:51 2014 -0500 gpfs_find_access_for_ion is BGQ-specific For PE and other not-bluegene GPFS platforms, provide the "build an access based on IO nodes" function only on Blue Gene. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 37d3fd8..3d11db9 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -422,6 +422,7 @@ void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) ADIOI_Assert(rc == 0); } +#ifdef BGQPLATFORM /* my_start, my_end: this processes file domain. coudd be -1,-1 for "no i/o" * fd_start, fd_end: arrays of length fd->hints->cb_nodes specifying all file domains */ int gpfs_find_access_for_ion(ADIO_File fd, @@ -466,7 +467,7 @@ int gpfs_find_access_for_ion(ADIO_File fd, ADIOI_Free(rank_to_ionode); return 1; } - +#endif // BGQPLATFORM /* If successful, error_code is set to MPI_SUCCESS. Otherwise an error @@ -554,6 +555,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype gpfs_wr_access_start(fd->fd_sys, st_loc, end_loc - st_loc); } ADIO_Offset st_loc_ion, end_loc_ion, needs_gpfs_access_cleanup=0; +#ifdef BGQPLATFORM if (getenv("ROMIO_GPFS_DECLARE_ION_ACCESS")!=NULL) { if (gpfs_find_access_for_ion(fd, st_loc, end_loc, fd_start, fd_end, &st_loc_ion, &end_loc_ion)) { @@ -561,6 +563,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype needs_gpfs_access_cleanup=1; } } +#endif MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX, fd->comm); http://git.mpich.org/mpich.git/commitdiff/ae5b26577bfcc2cb3034fccc2daba3138… commit ae5b26577bfcc2cb3034fccc2daba3138a264682 Author: Paul Coffman <pkcoff(a)us.ibm.com> Date: Mon Apr 14 16:48:39 2014 -0500 Assert on non-zero gpfs_fcntl an assertion is pretty heavy-handed here but if we have gone through the hoops to request these routines, we should know if they fail. Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index c052081..37d3fd8 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -397,6 +397,7 @@ void gpfs_wr_access_start(int fd, ADIO_Offset offset, ADIO_Offset length) take_locks.access.isWrite = 1; rc = gpfs_fcntl(fd, &take_locks); + ADIOI_Assert(rc == 0); } void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) @@ -418,6 +419,7 @@ void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) free_locks.free.length = length; rc = gpfs_fcntl(fd, &free_locks); + ADIOI_Assert(rc == 0); } /* my_start, my_end: this processes file domain. coudd be -1,-1 for "no i/o" http://git.mpich.org/mpich.git/commitdiff/8f6179fa8bdd050a0038ab1034a550a7b… commit 8f6179fa8bdd050a0038ab1034a550a7b7e922a5 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Thu Apr 10 16:01:52 2014 +0000 make event elements array a bit bigger typical convention is to use last element of enum "LAST_ITEM" as a marker for how big an array of said enum would be. the timing code though uses this last element. xl compiler caught us overrunning the array. diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c index 902c3ac..43236e0 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c @@ -40,8 +40,8 @@ int gpfsmpio_p2pcontig; int gpfsmpio_balancecontig; int gpfsmpio_devnullio; -double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST]; -double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST]; +double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST+1]; +double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; /* set internal variables for tuning environment variables */ /** \page mpiio_vars MPIIO Configuration diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h index 735eba6..c5b9c84 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h @@ -52,8 +52,9 @@ enum { GPFSMPIO_CIO_LAST }; -extern double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST]; -extern double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST]; +/* +1 because GPFSMPIO_CIO_LAST is actually used to say "zero this counter"" */ +extern double gpfsmpio_prof_cw [GPFSMPIO_CIO_LAST+1]; +extern double gpfsmpio_prof_cr [GPFSMPIO_CIO_LAST+1]; /* corresponds to environment variables to select optimizations and timing level */ http://git.mpich.org/mpich.git/commitdiff/3c9992ef6a70f6f977e74fe994c9b7e56… commit 3c9992ef6a70f6f977e74fe994c9b7e56d3b00c3 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Thu Apr 10 16:00:35 2014 +0000 Debugging code caused segfault this debugging code, when --enable-g=all is set, will exectue but not print anything. for some problems sizes/scale, the way it accesses the 128k'th element causes a segfault. diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c index 494ea12..938a133 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c @@ -1115,7 +1115,8 @@ static void ADIOI_R_Exchange_data_alltoallv( DBG_FPRINTF(stderr, "\ttails = %4d, %4d\n", stail, rtail ); if (nprocs_send) { DBG_FPRINTF(stderr, "\tall_send_buf = [%d]%2d,",0,all_send_buf[0]); - for (i=1; i<nprocs; i++) if(all_send_buf[(i-1)*131072]!=all_send_buf[i*131072]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i, all_send_buf [i*131072] ); } + /* someone at some point found it useful to look at the 128th kilobyte of data from each processor, but this segfaults in many situations if "all debugging" enabled */ + //for (i=1; i<nprocs; i++) if(all_send_buf[(i-1)*131072]!=all_send_buf[i*131072]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i, all_send_buf [i*131072] ); } } #endif http://git.mpich.org/mpich.git/commitdiff/8421f3a9c477188aae522caa23969c4ea… commit 8421f3a9c477188aae522caa23969c4ea2c68ea0 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Tue Feb 25 09:18:53 2014 -0600 an IO-node aware mode for gpfs hints Send a single access_range hint to the IO node in hopes this approach will make the gpfs_fcntl hints do something. diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 4bab0c1..c052081 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -17,6 +17,8 @@ #include "ad_gpfs.h" #include "ad_gpfs_aggrs.h" +#include <mpix.h> + #ifdef AGGREGATION_PROFILE #include "mpe.h" #endif @@ -33,6 +35,7 @@ #include <gpfs_fcntl.h> #endif +#include <limits.h> /* prototypes of functions used for collective writes only. */ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype datatype, int nprocs, int myrank, ADIOI_Access @@ -417,6 +420,50 @@ void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) rc = gpfs_fcntl(fd, &free_locks); } +/* my_start, my_end: this processes file domain. coudd be -1,-1 for "no i/o" + * fd_start, fd_end: arrays of length fd->hints->cb_nodes specifying all file domains */ +int gpfs_find_access_for_ion(ADIO_File fd, + ADIO_Offset my_start, ADIO_Offset my_end, + ADIO_Offset *fd_start, ADIO_Offset *fd_end, + ADIO_Offset *start, ADIO_Offset *end) +{ + int my_ionode = MPIX_IO_node_id(); + int *rank_to_ionode; + int i, nprocs, rank; + ADIO_Offset group_start=LLONG_MAX, group_end=0; + + MPI_Comm_size(fd->comm, &nprocs); + MPI_Comm_rank(fd->comm, &rank); + + rank_to_ionode = ADIOI_Calloc(nprocs, sizeof(int)); + MPI_Allgather(&my_ionode, 1, MPI_INT, rank_to_ionode, 1, MPI_INT, fd->comm); + + /* rank_to_ionode now contains a mapping from MPI rank to IO node */ + /* fd->hints->ranklist[] contains a list of MPI ranks that are aggregators */ + /* fd_start[] and fd_end[] contain a list of file domains. */ + + /* what we really want to do is take all the file domains associated + * with a given i/o node and find the begin/end of that range. + * + * Because gpfs_fcntl hints are expected to be released, we'll pass this + * start/end back to the caller, who will both declare and free this range + */ + if (my_start == -1 || my_end == -1) { + ADIOI_Free(rank_to_ionode); + return 0; /* no work to do */ + } + + for (i=0; i<fd->hints->cb_nodes; i++ ){ + if (my_ionode == rank_to_ionode[fd->hints->ranklist[i]] ) { + group_start = ADIOI_MIN(fd_start[i], group_start); + group_end = ADIOI_MAX(fd_end[i], group_end); + } + } + *start = group_start; + *end = group_end; + ADIOI_Free(rank_to_ionode); + return 1; +} @@ -798,6 +845,11 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype gpfs_wr_access_end(fd->fd_sys, st_loc_ion, end_loc_ion-st_loc_ion); needs_gpfs_access_cleanup=0; } + + if (needs_gpfs_access_cleanup) { + gpfs_wr_access_end(fd->fd_sys, end_loc_ion-st_loc_ion, st_loc_ion); + needs_gpfs_access_cleanup=0; + } unsetenv("LIBIOLOG_EXTRA_INFO"); } http://git.mpich.org/mpich.git/commitdiff/67bf775e9534ef87f1cb8dec536a1f297… commit 67bf775e9534ef87f1cb8dec536a1f2977379abc Author: Rob Latham <robl(a)mcs.anl.gov> Date: Thu Feb 20 13:47:57 2014 -0600 romio+gpfs: experiement with per-access hints these don't seem to help on vesta next up: sending individual range requests to the io node (once I figure out how to do that) diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c index 2ff9842..4bab0c1 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c @@ -26,6 +26,13 @@ #include <pthread.h> +#ifdef HAVE_GPFS_H +#include <gpfs.h> +#endif +#ifdef HAVE_GPFS_FCNTL_H +#include <gpfs_fcntl.h> +#endif + /* prototypes of functions used for collective writes only. */ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype datatype, int nprocs, int myrank, ADIOI_Access @@ -368,6 +375,49 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count, #endif } +void gpfs_wr_access_start(int fd, ADIO_Offset offset, ADIO_Offset length) +{ + int rc; + struct { + gpfsFcntlHeader_t header; + gpfsAccessRange_t access; + } take_locks; + + take_locks.header.totalLength = sizeof(take_locks); + take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION; + take_locks.header.fcntlReserved = 0; + + take_locks.access.structLen = sizeof(take_locks.access); + take_locks.access.structType = GPFS_ACCESS_RANGE; + take_locks.access.start = offset; + take_locks.access.length = length; + take_locks.access.isWrite = 1; + + rc = gpfs_fcntl(fd, &take_locks); +} + +void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length) +{ + int rc; + struct { + gpfsFcntlHeader_t header; + gpfsFreeRange_t free; + } free_locks; + + + free_locks.header.totalLength = sizeof(free_locks); + free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION; + free_locks.header.fcntlReserved = 0; + + free_locks.free.structLen = sizeof(free_locks.free); + free_locks.free.structType = GPFS_FREE_RANGE; + free_locks.free.start = offset; + free_locks.free.length = length; + + rc = gpfs_fcntl(fd, &free_locks); +} + + /* If successful, error_code is set to MPI_SUCCESS. Otherwise an error @@ -451,6 +501,17 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype if ((st_loc==-1) && (end_loc==-1)) { ntimes = 0; /* this process does no writing. */ } + if (ntimes != 0 && getenv("ROMIO_GPFS_DECLARE_ACCESS")!=NULL) { + gpfs_wr_access_start(fd->fd_sys, st_loc, end_loc - st_loc); + } + ADIO_Offset st_loc_ion, end_loc_ion, needs_gpfs_access_cleanup=0; + if (getenv("ROMIO_GPFS_DECLARE_ION_ACCESS")!=NULL) { + if (gpfs_find_access_for_ion(fd, st_loc, end_loc, fd_start, fd_end, + &st_loc_ion, &end_loc_ion)) { + gpfs_wr_access_start(fd->fd_sys, st_loc_ion, end_loc_ion-st_loc_ion); + needs_gpfs_access_cleanup=1; + } + } MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX, fd->comm); @@ -729,6 +790,14 @@ static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype ADIOI_Free(curr_to_proc); ADIOI_Free(done_to_proc); + if (ntimes != 0 && getenv("ROMIO_GPFS_DECLARE_ACCESS")!=NULL) { + gpfs_wr_access_end(fd->fd_sys, st_loc, end_loc-st_loc); + } + + if (needs_gpfs_access_cleanup) { + gpfs_wr_access_end(fd->fd_sys, st_loc_ion, end_loc_ion-st_loc_ion); + needs_gpfs_access_cleanup=0; + } unsetenv("LIBIOLOG_EXTRA_INFO"); } http://git.mpich.org/mpich.git/commitdiff/faad606c90bcd14204021bbd0ab0de013… commit faad606c90bcd14204021bbd0ab0de01351da779 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Mon Jan 27 19:48:54 2014 +0000 romio+gpfs: experiment with gpfs_fcntl hints When many processes write to a gpfs file, even with nicely aligned blocks, we see immense variation in access times. Could that variation be mitigated if we drop locks on first access? next experiment (not yet implemented): what if we pre-declare access/ranges when file domains computed? diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_aggrs.h b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_aggrs.h index 4d0b0da..1b6215e 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_aggrs.h +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_aggrs.h @@ -18,6 +18,10 @@ #include "adio.h" #include <sys/stat.h> + +#ifdef HAVE_GPFS_H +#include <gpfs.h> +#endif /* overriding ADIOI_Calc_file_domains() to apply 'aligned file domain partitioning'. */ diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c index 99b2c1b..4e236b6 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c @@ -19,6 +19,39 @@ #include <sys/types.h> #include <unistd.h> +#ifdef HAVE_GPFS_H +#include <gpfs.h> +#endif +#ifdef HAVE_GPFS_FCNTL_H +#include <gpfs_fcntl.h> +#endif + +#ifdef HAVE_GPFS_FCNTL_H +void gpfs_free_all_locks(int fd) +{ + int rc; + struct { + gpfsFcntlHeader_t header; + gpfsFreeRange_t release; + } release_all; + + release_all.header.totalLength = sizeof(release_all); + release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION; + release_all.header.fcntlReserved = 0; + + release_all.release.structLen = sizeof(release_all.release); + release_all.release.structType = GPFS_FREE_RANGE; + release_all.release.start = 0; + release_all.release.length = 0; + + rc = gpfs_fcntl(fd, &release_all); + if (rc != 0) { + DBGV_FPRINTF(stderr,"GPFS fcntl release failed with rc=%d, errno=%d\n", + rc,errno); + } +} +#endif + void ADIOI_GPFS_Open(ADIO_File fd, int *error_code) { @@ -101,6 +134,14 @@ void ADIOI_GPFS_Open(ADIO_File fd, int *error_code) #ifdef ADIOI_MPE_LOGGING MPE_Log_event(ADIOI_MPE_stat_b, 0, NULL); #endif + +#ifdef HAVE_GPFS_FCNTL_H + /* in parallel workload, might be helpful to immediately release block + * tokens. Or, system call overhead will outweigh any benefits... */ + if (getenv("ROMIO_GPFS_FREE_LOCKS")!=NULL) + gpfs_free_all_locks(fd->fd_sys); + +#endif } if (fd->fd_sys == -1) { diff --git a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.h b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.h index d6baaed..b154722 100644 --- a/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.h +++ b/src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.h @@ -20,6 +20,12 @@ #include "adio.h" #include <sys/stat.h> +#ifdef HAVE_GPFS_H +#include <gpfs.h> +#endif +#if !defined(GPFS_SUPER_MAGIC) + #define GPFS_SUPER_MAGIC (0x47504653) +#endif /* generate a list of I/O aggregators that utilizes BG-PSET orginization. */ int ADIOI_BG_gen_agg_ranklist(ADIO_File fd, int n_aggrs_per_pset); diff --git a/src/mpi/romio/configure.ac b/src/mpi/romio/configure.ac index 484556d..df13053 100644 --- a/src/mpi/romio/configure.ac +++ b/src/mpi/romio/configure.ac @@ -973,6 +973,11 @@ fi AS_IF([test -n "$file_system_gpfs"], [SYSDEP_INC=-I${prefix}/include], [SYSDEP_INC=]) +AC_CHECK_HEADERS([gpfs.h gpfs_fcntl.h]) +AS_IF([test "$ac_cv_header_gpfs_h" = "yes" -o "$ac_cv_header_gpfs_fcntl_h" = "yes"], [ + AC_SEARCH_LIBS([gpfs_fcntl], [gpfs], [], + [AC_MSG_ERROR([Library containing gpfs_fcntl symbols not found])]) +]) # Check for presence and characteristics of async. I/O calls if # not disabled. ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/ad_gpfs/ad_gpfs_aggrs.h | 4 + src/mpi/romio/adio/ad_gpfs/ad_gpfs_open.c | 41 ++++++++ src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c | 3 +- src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.c | 16 +++- src/mpi/romio/adio/ad_gpfs/ad_gpfs_tuning.h | 6 +- src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c | 130 +++++++++++++++++++++++++ src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.c | 140 +++++++++++++++++++++++++-- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_aggrs.h | 6 + src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.c | 81 +++++++++++++++- src/mpi/romio/adio/ad_gpfs/bg/ad_bg_pset.h | 6 + src/mpi/romio/configure.ac | 5 + src/mpi/romio/mpi-io/open.c | 11 ++- 12 files changed, 432 insertions(+), 17 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-266-g659053c
by noreply＠mpich.org 23 May '14

23 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 659053cbe6a35e78ad166d72ce927de7bc69f0d4 (commit) from ec32bcfe3aed767d598a5df196bed0f9fcd31570 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/659053cbe6a35e78ad166d72ce927de7b… commit 659053cbe6a35e78ad166d72ce927de7bc69f0d4 Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Fri May 23 09:35:11 2014 -0500 remove check for strtoll Nemesis no longer requires this function when using sysv shared memory. Signed-off-by: Sangmin Seo <sseo(a)anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/subconfigure.m4 index e636b69..558cbdf 100644 --- a/src/mpid/ch3/channels/nemesis/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/subconfigure.m4 @@ -191,10 +191,6 @@ if test "$with_shared_memory" != "mmap" -a "$with_shared_memory" != "sysv"; then AC_MSG_ERROR([cannot support shared memory: need either sysv shared memory functions or mmap in order to support shared memory]) fi -if test "$found_sysv_shm_funcs" = yes ; then - AC_CHECK_FUNCS(strtoll, , AC_MSG_ERROR([cannot find strtoll function needed by sysv shared memory implementation])) -fi - AC_ARG_ENABLE(nemesis-shm-collectives, [--enable-nemesis-shm-collectives - enables use of shared memory for collective comunication within a node], AC_DEFINE(ENABLED_SHM_COLLECTIVES, 1, [Define to enable shared-memory collectives])) ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/channels/nemesis/subconfigure.m4 | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-265-gec32bcf
by noreply＠mpich.org 22 May '14

22 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via ec32bcfe3aed767d598a5df196bed0f9fcd31570 (commit) from ad09da4a3394eb59e38c805b167fe047c55dce4c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/ec32bcfe3aed767d598a5df196bed0f9f… commit ec32bcfe3aed767d598a5df196bed0f9fcd31570 Author: Michael Blocksome <blocksom(a)us.ibm.com> Date: Thu May 22 15:16:22 2014 -0500 pamid: use shared memory configure option m4 macro diff --git a/src/mpid/pamid/subconfigure.m4 b/src/mpid/pamid/subconfigure.m4 index ffd8e1a..f6128da 100644 --- a/src/mpid/pamid/subconfigure.m4 +++ b/src/mpid/pamid/subconfigure.m4 @@ -92,6 +92,12 @@ dnl Set the pamid platform define. dnl PAC_APPEND_FLAG([-D__${pamid_platform}__], [CPPFLAGS]) +dnl Check for available shared memory functions +PAC_ARG_SHARED_MEMORY +if test "$with_shared_memory" != "mmap" -a "$with_shared_memory" != "sysv"; then + AC_MSG_ERROR([cannot support shared memory: need either sysv shared memory functions or mmap in order to support shared memory]) +fi + dnl dnl This configure option allows "sandbox" bgq system software to be used. dnl ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/subconfigure.m4 | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1-264-gad09da4
by noreply＠mpich.org 22 May '14

22 May '14

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via ad09da4a3394eb59e38c805b167fe047c55dce4c (commit) from 25e3a95390be3a7c79e65b21a67b8a1154d57fa2 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/ad09da4a3394eb59e38c805b167fe047c… commit ad09da4a3394eb59e38c805b167fe047c55dce4c Author: Ken Raffenetti <raffenet(a)mcs.anl.gov> Date: Wed May 21 17:27:30 2014 -0500 moved shared memory configure option to m4 macro Other parts of the code might want to use shared memory. This commit extracts our checks for available functions into a macro for easy re-use outside of nemesis. Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/confdb/aclocal_shm.m4 b/confdb/aclocal_shm.m4 new file mode 100644 index 0000000..d2cbaad --- /dev/null +++ b/confdb/aclocal_shm.m4 @@ -0,0 +1,49 @@ +dnl +dnl Definitions for using shared memory +dnl + +dnl/*D +dnl PAC_ARG_SHARED_MEMORY - add --with-shared-memory=kind to configure +dnl +dnl Synopsis: +dnl PAC_ARG_SHARED_MEMORY +dnl +dnl Output effects: +dnl Adds '--with-shared-memory' to the command line. Checks for available +dnl shared memory functionality. +dnl +dnl Supported values of 'kind' include \: +dnl+ auto - default +dnl. mmap - use mmap and munmap +dnl- sysv - use sysv shared memory functions +dnl D*/ +AC_DEFUN([PAC_ARG_SHARED_MEMORY],[ + +# check how to allocate shared memory +AC_ARG_WITH(shared-memory, + AC_HELP_STRING([--with-shared-memory[=auto|sysv|mmap]], [create shared memory using sysv or mmap (default is auto)]),, + with_shared_memory=auto) + +if test "$with_shared_memory" = auto -o "$with_shared_memory" = mmap; then + found_mmap_funcs=yes + AC_CHECK_FUNCS(mmap munmap, , found_mmap_funcs=no) + if test "$found_mmap_funcs" = yes ; then + with_shared_memory=mmap + AC_DEFINE(USE_MMAP_SHM,1,[Define if we have sysv shared memory]) + AC_MSG_NOTICE([Using a memory-mapped file for shared memory]) + elif test "$with_shared_memory" = mmap ; then + AC_MSG_ERROR([cannot support shared memory: mmap() or munmap() not found]) + fi +fi +if test "$with_shared_memory" = auto -o "$with_shared_memory" = sysv; then + found_sysv_shm_funcs=yes + AC_CHECK_FUNCS(shmget shmat shmctl shmdt, , found_sysv_shm_funcs=no) + if test "$found_sysv_shm_funcs" = yes ; then + with_shared_memory=sysv + AC_DEFINE(USE_SYSV_SHM,1,[Define if we have sysv shared memory]) + AC_MSG_NOTICE([Using SYSV shared memory]) + elif test "$with_shared_memory" = sysv ; then + AC_MSG_ERROR([cannot support shared memory: sysv shared memory functions functions not found]) + fi +fi +]) diff --git a/src/mpid/ch3/channels/nemesis/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/subconfigure.m4 index 9c69219..e636b69 100644 --- a/src/mpid/ch3/channels/nemesis/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/subconfigure.m4 @@ -185,32 +185,10 @@ AC_CHECK_FUNCS(mkstemp) AC_CHECK_FUNCS(rand) AC_CHECK_FUNCS(srand) -# check how to allocate shared memory -AC_ARG_WITH(shared-memory, [--with-shared-memory[=auto|sysv|mmap] - create shared memory using sysv or mmap (default is auto)],, - with_shared_memory=auto) - -if test "$with_shared_memory" = auto -o "$with_shared_memory" = mmap; then - found_mmap_funcs=yes - AC_CHECK_FUNCS(mmap munmap, , found_mmap_funcs=no) - if test "$found_mmap_funcs" = yes ; then - with_shared_memory=mmap - AC_DEFINE(USE_MMAP_SHM,1,[Define if we have sysv shared memory]) - AC_MSG_NOTICE([Using a memory-mapped file for shared memory]) - elif test "$with_shared_memory" = mmap ; then - AC_MSG_ERROR([cannot support shared memory: mmap() or munmap() not found]) - fi -fi -if test "$with_shared_memory" = auto -o "$with_shared_memory" = sysv; then - found_sysv_shm_funcs=yes - AC_CHECK_FUNCS(shmget shmat shmctl shmdt, , found_sysv_shm_funcs=no) - if test "$found_sysv_shm_funcs" = yes ; then - AC_DEFINE(USE_SYSV_SHM,1,[Define if we have sysv shared memory]) - AC_MSG_NOTICE([Using SYSV shared memory]) - elif test "$with_shared_memory" = sysv ; then - AC_MSG_ERROR([cannot support shared memory: sysv shared memory functions functions not found]) - else - AC_MSG_ERROR([cannot support shared memory: need either sysv shared memory functions or mmap in order to support shared memory]) - fi +# Check for available shared memory functions +PAC_ARG_SHARED_MEMORY +if test "$with_shared_memory" != "mmap" -a "$with_shared_memory" != "sysv"; then + AC_MSG_ERROR([cannot support shared memory: need either sysv shared memory functions or mmap in order to support shared memory]) fi if test "$found_sysv_shm_funcs" = yes ; then ----------------------------------------------------------------------- Summary of changes: confdb/aclocal_shm.m4 | 49 +++++++++++++++++++++++++ src/mpid/ch3/channels/nemesis/subconfigure.m4 | 30 ++------------- 2 files changed, 53 insertions(+), 26 deletions(-) create mode 100644 confdb/aclocal_shm.m4 hooks/post-receive -- MPICH primary repository

1 0