This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 7d44307f269cae96118beb19760221aff99bd74a (commit) from 2de997d9b3e94bad01d5f46d76f163d71e2bd7bd (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/7d44307f269cae96118beb19760221aff9... commit 7d44307f269cae96118beb19760221aff99bd74a Author: Rob Latham <[email protected]> Date: Thu Jun 27 10:25:37 2013 -0500 partial fix for tt 1742 It would appear that we can get more than two gigs worth of data down to the write(2) and read(2) system call, but those syscalls fail to process all of it. Perfectly valid behavior for unix to return less than requested, but until now, ROMIO never dealt with it. Still to do: update Blue Gene drivers closes ticket #1742 , since that ticket is only asking about I/O to less than two gigs. Lots and Lots more work needed for larger than two gigs. diff --git a/src/mpi/romio/adio/common/ad_read.c b/src/mpi/romio/adio/common/ad_read.c index 3cea7ed..1961b16 100644 --- a/src/mpi/romio/adio/common/ad_read.c +++ b/src/mpi/romio/adio/common/ad_read.c @@ -22,15 +22,16 @@ void ADIOI_GEN_ReadContig(ADIO_File fd, void *buf, int count, off_t err_lseek = -1; ssize_t err = -1; int datatype_size; - ADIO_Offset len; + ADIO_Offset len, bytes_xfered=0; + size_t rd_count; static char myname[] = "ADIOI_GEN_READCONTIG"; + char *p; #ifdef AGGREGATION_PROFILE MPE_Log_event (5034, 0, NULL); #endif MPI_Type_size(datatype, &datatype_size); len = (ADIO_Offset)datatype_size * (ADIO_Offset)count; - ADIOI_Assert(len == (unsigned int) len); /* read takes an unsigned int parm */ if (file_ptr_type == ADIO_INDIVIDUAL) { offset = fd->fp_ind; @@ -57,33 +58,46 @@ void ADIOI_GEN_ReadContig(ADIO_File fd, void *buf, int count, /* --END ERROR HANDLING-- */ } + p=buf; + while (bytes_xfered < len) { #ifdef ADIOI_MPE_LOGGING - MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); + MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); #endif - err = read(fd->fd_sys, buf, (unsigned int)len); + rd_count = len - bytes_xfered; + err = read(fd->fd_sys, p, rd_count); + /* --BEGIN ERROR HANDLING-- */ + if (err == -1) { + *error_code = MPIO_Err_create_code(MPI_SUCCESS, + MPIR_ERR_RECOVERABLE, + myname, __LINE__, + MPI_ERR_IO, "**io", + "**io %s", strerror(errno)); + fd->fp_sys_posn = -1; + return; + } + /* --END ERROR HANDLING-- */ + if (err == 0) { + /* end of file */ + break; + } + #ifdef ADIOI_MPE_LOGGING - MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); + MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); #endif - /* --BEGIN ERROR HANDLING-- */ - if (err == -1) { - *error_code = MPIO_Err_create_code(MPI_SUCCESS, - MPIR_ERR_RECOVERABLE, - myname, __LINE__, - MPI_ERR_IO, "**io", - "**io %s", strerror(errno)); - fd->fp_sys_posn = -1; - return; + bytes_xfered += err; + p += err; } - /* --END ERROR HANDLING-- */ - fd->fp_sys_posn = offset + err; + fd->fp_sys_posn = offset + bytes_xfered; if (file_ptr_type == ADIO_INDIVIDUAL) { - fd->fp_ind += err; + fd->fp_ind += bytes_xfered; } #ifdef HAVE_STATUS_SET_BYTES - if (err != -1) MPIR_Status_set_bytes(status, datatype, err); + /* what if we only read half a datatype? */ + /* bytes_xfered could be larger than int */ + if (err != -1) MPIR_Status_set_bytes(status, datatype, bytes_xfered); #endif *error_code = MPI_SUCCESS; diff --git a/src/mpi/romio/adio/common/ad_write.c b/src/mpi/romio/adio/common/ad_write.c index eeba5da..6f01aff 100644 --- a/src/mpi/romio/adio/common/ad_write.c +++ b/src/mpi/romio/adio/common/ad_write.c @@ -22,8 +22,10 @@ void ADIOI_GEN_WriteContig(ADIO_File fd, const void *buf, int count, off_t err_lseek = -1; ssize_t err = -1; int datatype_size; - ADIO_Offset len; + ADIO_Offset len, bytes_xfered=0; + size_t wr_count; static char myname[] = "ADIOI_GEN_WRITECONTIG"; + char * p; #ifdef AGGREGATION_PROFILE MPE_Log_event (5036, 0, NULL); @@ -31,7 +33,6 @@ void ADIOI_GEN_WriteContig(ADIO_File fd, const void *buf, int count, MPI_Type_size(datatype, &datatype_size); len = (ADIO_Offset)datatype_size * (ADIO_Offset)count; - ADIOI_Assert(len == (unsigned int) len); /* read takes an unsigned int parm */ if (file_ptr_type == ADIO_INDIVIDUAL) { offset = fd->fp_ind; @@ -58,33 +59,40 @@ void ADIOI_GEN_WriteContig(ADIO_File fd, const void *buf, int count, /* --END ERROR HANDLING-- */ } + p = (char *)buf; + while (bytes_xfered < len) { #ifdef ADIOI_MPE_LOGGING - MPE_Log_event( ADIOI_MPE_write_a, 0, NULL ); + MPE_Log_event( ADIOI_MPE_write_a, 0, NULL ); #endif - err = write(fd->fd_sys, buf, (unsigned int)len); + wr_count = len - bytes_xfered; + err = write(fd->fd_sys, p, wr_count); + /* --BEGIN ERROR HANDLING-- */ + if (err == -1) { + *error_code = MPIO_Err_create_code(MPI_SUCCESS, + MPIR_ERR_RECOVERABLE, + myname, __LINE__, + MPI_ERR_IO, "**io", + "**io %s", strerror(errno)); + fd->fp_sys_posn = -1; + return; + } + /* --END ERROR HANDLING-- */ #ifdef ADIOI_MPE_LOGGING - MPE_Log_event( ADIOI_MPE_write_b, 0, NULL ); + MPE_Log_event( ADIOI_MPE_write_b, 0, NULL ); #endif - /* --BEGIN ERROR HANDLING-- */ - if (err == -1) { - *error_code = MPIO_Err_create_code(MPI_SUCCESS, - MPIR_ERR_RECOVERABLE, - myname, __LINE__, - MPI_ERR_IO, "**io", - "**io %s", strerror(errno)); - fd->fp_sys_posn = -1; - return; + bytes_xfered += err; + p += err; } - /* --END ERROR HANDLING-- */ - fd->fp_sys_posn = offset + err; + fd->fp_sys_posn = offset + bytes_xfered; if (file_ptr_type == ADIO_INDIVIDUAL) { - fd->fp_ind += err; + fd->fp_ind += bytes_xfered; } #ifdef HAVE_STATUS_SET_BYTES - if (err != -1 && status) MPIR_Status_set_bytes(status, datatype, err); + /* bytes_xfered could be larger than int */ + if (err != -1 && status) MPIR_Status_set_bytes(status, datatype, bytes_xfered); #endif *error_code = MPI_SUCCESS; diff --git a/src/mpi/romio/configure.ac b/src/mpi/romio/configure.ac index a1d0a5e..298ab99 100644 --- a/src/mpi/romio/configure.ac +++ b/src/mpi/romio/configure.ac @@ -706,6 +706,10 @@ fi # AC_C_INLINE +AC_TYPE_SIZE_T +AC_TYPE_SSIZE_T +AC_TYPE_OFF_T + # Header files # Find the CPP before the header check AC_PROG_CPP ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/adio/common/ad_read.c | 50 +++++++++++++++++++++------------ src/mpi/romio/adio/common/ad_write.c | 44 +++++++++++++++++------------ src/mpi/romio/configure.ac | 4 +++ 3 files changed, 62 insertions(+), 36 deletions(-) hooks/post-receive -- MPICH primary repository