This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 6a954d01628b1443336bee7e197d91d56690d8f6 (commit) via 4aba817fda95da257258befed3fa34167ee44507 (commit) via b187ee20f4167555ed752601ba56f62878c02fe4 (commit) via 237d91bf62040f46f32ecf5f65a475c578bd6b9d (commit) from 39ed84ae13cbed02f561144a0b509c052339377d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/6a954d01628b1443336bee7e197d91d566... commit 6a954d01628b1443336bee7e197d91d56690d8f6 Author: Dave Goodell <[email protected]> Date: Fri Jan 25 16:50:49 2013 -0600 add `xfail=ticketXYZ` annotations to failing tests These five tests are currently failing on a regular basis with MPICH. All need to be dealt with at some point, but we cannot fix them right now and they make it hard to spot new failures in the tests. references #1767, #1788 Reviewed-by: dinan diff --git a/test/mpi/datatype/testlist b/test/mpi/datatype/testlist index ccef43d..755b25b 100644 --- a/test/mpi/datatype/testlist +++ b/test/mpi/datatype/testlist @@ -48,5 +48,5 @@ get-elements 1 hindexed_block 1 mpiversion=3.0 hindexed_block_contents 1 mpiversion=3.0 longdouble 1 -large-count 1 mpiversion=3.0 +large-count 1 mpiversion=3.0 xfail=ticket1767 cxx-types 1 mpiversion=3.0 diff --git a/test/mpi/perf/testlist b/test/mpi/perf/testlist index 503729f..b81ea53 100644 --- a/test/mpi/perf/testlist +++ b/test/mpi/perf/testlist @@ -1,10 +1,10 @@ transp-datatype 2 sendrecvl 2 -twovec 1 +twovec 1 xfail=ticket1788 dtpack 1 -nestvec 1 -nestvec2 1 -indexperf 1 +nestvec 1 xfail=ticket1788 +nestvec2 1 xfail=ticket1788 +indexperf 1 xfail=ticket1788 non_zero_root 4 timer 1 # The commcreatep test looks at how communicator creation scales with group http://git.mpich.org/mpich.git/commitdiff/4aba817fda95da257258befed3fa34167e... commit 4aba817fda95da257258befed3fa34167ee44507 Author: Dave Goodell <[email protected]> Date: Fri Jan 25 14:51:17 2013 -0600 TAP: add "xfail" support This is used to indicate tests that are correct, but functionality that is expected to fail in the current version of the MPI implementation. This is expressed in `testlist` files with an argument string like `xfail=ticketXYZ` or `xfail=REASON` after the program name. Unfortunately, spaces, quotes, and `#` characters are not permitted in the format at this time, due to the primitive nature of the testfile parser code. If strict MPI testing is requested (by configuring the test suite with `--enable-strictmpi`) then the `xfail=BLAH` arguments are ignored under the assumption that they are only accurate for the current version of the MPICH implementation. No xfail support has been added to the XML output at this time. Reviewed-by: dinan diff --git a/test/mpi/runtests.in b/test/mpi/runtests.in index 3046cc8..56b6e39 100644 --- a/test/mpi/runtests.in +++ b/test/mpi/runtests.in @@ -362,6 +362,7 @@ sub RunList { my $requiresMPIX = ""; my $progEnv = ""; my $mpiVersion = ""; + my $xfail = ""; if ($#args >= 1) { $np = $args[1]; } # Process the key=value arguments for (my $i=2; $i <= $#args; $i++) { @@ -395,6 +396,12 @@ sub RunList { elsif ($key eq "mpix") { $requiresMPIX = $value } + elsif ($key eq "xfail") { + if ($value eq "") { + print STDERR "\"xfail=\" requires an argument\n"; + } + $xfail = $value; + } else { print STDERR "Unrecognized key $key in $listfileSource\n"; } @@ -441,6 +448,13 @@ sub RunList { next; } + if (lc($testIsStrict) eq "true") { + # Strict MPI testing was requested, so assume that a non-MPICH MPI + # implementation is being tested and the "xfail" implementation + # assumptions do not hold. + $xfail = ''; + } + if (lc($requiresMPIX) eq "true" && lc($MPIHasMPIX) eq "no") { unless (-d $programname) { SkippedTest($programname, $np, $workdir, "tests MPIX extensions, MPIX testing disabled"); @@ -455,19 +469,19 @@ sub RunList { } else { $total_run++; - if (&BuildMPIProgram( $programname ) == 0) { + if (&BuildMPIProgram( $programname, $xfail ) == 0) { if ($batchRun == 1) { &AddMPIProgram( $programname, $np, $ResultTest, $InitForRun, $timeLimit, $progArgs, - $progEnv, $mpiexecArgs ); + $progEnv, $mpiexecArgs, $xfail ); } else { &RunMPIProgram( $programname, $np, $ResultTest, $InitForRun, $timeLimit, $progArgs, - $progEnv, $mpiexecArgs ); + $progEnv, $mpiexecArgs, $xfail ); } } - else { + elsif ($xfail ne '') { # We expected to run this program, so failure to build # is an error $found_error = 1; @@ -520,7 +534,7 @@ sub ProcessImplicitList { if ($programname eq "runtests") { next; } # Ignore self if (-x $programname) { $total_run++; - &RunMPIProgram( $programname, $np_default, "", "", "", "", "", "" ); + &RunMPIProgram( $programname, $np_default, "", "", "", "", "", "", "" ); } } close PGMS; @@ -540,8 +554,8 @@ sub ProcessImplicitList { if (! -s $programname) { next; } $programname =~ s/\.c//; $total_run++; - if (&BuildMPIProgram( $programname ) == 0) { - &RunMPIProgram( $programname, $np_default, "", "", "", "", "", "" ); + if (&BuildMPIProgram( $programname, "") == 0) { + &RunMPIProgram( $programname, $np_default, "", "", "", "", "", "", "" ); } else { # We expected to run this program, so failure to build @@ -562,7 +576,7 @@ sub ProcessImplicitList { # If the 3rd arg is not present, the a default that simply checks that the # return status is 0 and that the output is " No Errors" is used. sub RunMPIProgram { - my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs) = @_; + my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs,$xfail) = @_; my $found_error = 0; my $found_noerror = 0; my $inline = ""; @@ -655,10 +669,10 @@ sub RunMPIProgram { } } if ($found_error) { - &RunTestFailed( $programname, $np, $curdir, $inline ); + &RunTestFailed( $programname, $np, $curdir, $inline, $xfail ); } else { - &RunTestPassed( $programname, $np, $curdir ); + &RunTestPassed( $programname, $np, $curdir, $xfail ); } &RunPostMsg( $programname, $np, $curdir ); } @@ -666,7 +680,7 @@ sub RunMPIProgram { # This version simply writes the mpiexec command out, with the output going # into a file, and recording the output status of the run. sub AddMPIProgram { - my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs) = @_; + my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs, $xfail) = @_; if (! -x $programname) { print STDERR "Could not find $programname!"; @@ -734,7 +748,8 @@ sub AddMPIProgram { # # Return value is 0 on success, non zero on failure sub BuildMPIProgram { - my $programname = $_[0]; + my $programname = shift; + my $xfail = shift; my $rc = 0; if ($verbose) { print STDERR "making $programname\n"; } if (! -x $programname) { $remove_this_pgm = 1; } @@ -752,7 +767,7 @@ sub BuildMPIProgram { # in the summary file (which is otherwise written by the # RunMPIProgram step) &RunPreMsg( $programname, $np, $curdir ); - &RunTestFailed( $programname, $np, $curdir, "Failed to build $programname; $output" ); + &RunTestFailed( $programname, $np, $curdir, "Failed to build $programname; $output", $xfail ); &RunPostMsg( $programname, $np, $curdir ); } return $rc; @@ -966,12 +981,16 @@ sub RunPostMsg { } } sub RunTestPassed { - my ($programname, $np, $workdir) = @_; + my ($programname, $np, $workdir, $xfail) = @_; if ($xmloutput) { print XMLOUT "<STATUS>pass</STATUS>$newline"; } if ($tapoutput) { - print TAPOUT "ok ${total_run} - $workdir/$programname $np\n"; + my $xfailstr = ''; + if ($xfail ne '') { + $xfailstr = " # TODO $xfail"; + } + print TAPOUT "ok ${total_run} - $workdir/$programname ${np}${xfailstr}\n"; } } sub RunTestFailed { @@ -979,6 +998,7 @@ sub RunTestFailed { my $np = shift; my $workdir = shift; my $output = shift; + my $xfail = shift; if ($xmloutput) { my $xout = $output; @@ -994,7 +1014,11 @@ sub RunTestFailed { } if ($tapoutput) { - print TAPOUT "not ok ${total_run} - $workdir/$programname $np\n"; + my $xfailstr = ''; + if ($xfail ne '') { + $xfailstr = " # TODO $xfail"; + } + print TAPOUT "not ok ${total_run} - $workdir/$programname ${np}${xfailstr}\n"; print TAPOUT " ---\n"; print TAPOUT " Directory: $workdir\n"; print TAPOUT " File: $programname\n"; http://git.mpich.org/mpich.git/commitdiff/b187ee20f4167555ed752601ba56f62878... commit b187ee20f4167555ed752601ba56f62878c02fe4 Author: Dave Goodell <[email protected]> Date: Fri Jan 25 14:50:14 2013 -0600 runtests: permit arbitrary capitalization in args At Jim's request... Reviewed-by: dinan diff --git a/test/mpi/runtests.in b/test/mpi/runtests.in index 7eec47f..3046cc8 100644 --- a/test/mpi/runtests.in +++ b/test/mpi/runtests.in @@ -434,14 +434,14 @@ sub RunList { } # Check whether strict is required by MPI but not by the # test (use strict=false for tests that use non-standard extensions) - if ($requiresStrict eq "false" && $testIsStrict eq "true") { + if (lc($requiresStrict) eq "false" && lc($testIsStrict) eq "true") { unless (-d $programname) { SkippedTest($programname, $np, $workdir, "non-strict test, strict MPI mode requested"); } next; } - if ($requiresMPIX eq "true" && $MPIHasMPIX eq "no") { + if (lc($requiresMPIX) eq "true" && lc($MPIHasMPIX) eq "no") { unless (-d $programname) { SkippedTest($programname, $np, $workdir, "tests MPIX extensions, MPIX testing disabled"); } http://git.mpich.org/mpich.git/commitdiff/237d91bf62040f46f32ecf5f65a475c578... commit 237d91bf62040f46f32ecf5f65a475c578bd6b9d Author: Dave Goodell <[email protected]> Date: Fri Jan 25 13:21:06 2013 -0600 TAP: add "# SKIP" directive support This way tests that were skipped because of strictness or version constraints will show up in the summary.tap output file. Reviewed-by: dinan diff --git a/test/mpi/runtests.in b/test/mpi/runtests.in index e71a978..7eec47f 100644 --- a/test/mpi/runtests.in +++ b/test/mpi/runtests.in @@ -47,7 +47,8 @@ $testIsStrict = "@MPI_IS_STRICT@"; $MPIhasMPIX = "@MPI_HAS_MPIX@"; $np_arg = "-n"; # Name of argument to specify the number of processes $err_count = 0; # Number of programs that failed. -$total_count = 0; # Number of programs tested +$total_run = 0; # Number of programs tested +$total_seen = 0; # Number of programs considered for testing $np_default = 2; # Default number of processes to use $np_max = -1; # Maximum number of processes to use (overrides any # value in the test list files. -1 is Infinity @@ -251,7 +252,7 @@ if ($xmloutput && $closeXMLOutput) { } if ($tapoutput) { - print TAPOUT "1..$total_count"; + print TAPOUT "1..$total_seen"; close TAPOUT; } @@ -262,13 +263,13 @@ if ($batchRun) { } else { if ($err_count) { - print "$err_count tests failed out of $total_count\n"; + print "$err_count tests failed out of $total_run\n"; if ($xmloutput) { print "Details in $xmlfullfile\n"; } } else { - print " All $total_count tests passed!\n"; + print " All $total_run tests passed!\n"; } if ($tapoutput) { print "TAP formatted results in $tapfullfile\n"; @@ -400,33 +401,60 @@ sub RunList { } } + # skip empty lines + if ($programname eq "") { next; } + + if ($np eq "") { $np = $np_default; } + if ($np_max > 0 && $np > $np_max) { $np = $np_max; } + + # allows us to accurately output TAP test numbers without disturbing the + # original totals that have traditionally been reported + # + # These "unless" blocks are ugly, but permit us to honor skipping + # criteria for directories as well without counting directories as tests + # in our XML/TAP output. + unless (-d $programname) { + $total_seen++; + } + # If a minimum MPI version is specified, check against the # available MPI. If the version is unknown, we ignore this # test (thus, all tests will be run). if ($mpiVersion ne "" && $MPIMajorVersion ne "unknown" && $MPIMinorVersion ne "unknown") { my ($majorReq,$minorReq) = split(/\./,$mpiVersion); - if ($majorReq > $MPIMajorVersion) { next; } - if ($majorReq == $MPIMajorVersion && - $minorReq > $MPIMinorVersion) { next; } + if ($majorReq > $MPIMajorVersion or + ($majorReq == $MPIMajorVersion && $minorReq > $MPIMinorVersion)) + { + unless (-d $programname) { + SkippedTest($programname, $np, $workdir, "requires MPI version $mpiVersion"); + } + next; + } } # Check whether strict is required by MPI but not by the # test (use strict=false for tests that use non-standard extensions) - if ($requiresStrict eq "false" && $testIsStrict eq "true") { next; } + if ($requiresStrict eq "false" && $testIsStrict eq "true") { + unless (-d $programname) { + SkippedTest($programname, $np, $workdir, "non-strict test, strict MPI mode requested"); + } + next; + } - if ($requiresMPIX eq "true" && $MPIHasMPIX eq "no") { next; } + if ($requiresMPIX eq "true" && $MPIHasMPIX eq "no") { + unless (-d $programname) { + SkippedTest($programname, $np, $workdir, "tests MPIX extensions, MPIX testing disabled"); + } + next; + } - if ($np eq "") { $np = $np_default; } - if ($np_max > 0 && $np > $np_max) { $np = $np_max; } - # skip empty lines - if ($programname eq "") { next; } if (-d $programname) { # If a directory, go into the that directory and # look for a new list file &ProcessDir( $programname, $listfile ); } else { - $total_count++; + $total_run++; if (&BuildMPIProgram( $programname ) == 0) { if ($batchRun == 1) { &AddMPIProgram( $programname, $np, $ResultTest, @@ -491,7 +519,7 @@ sub ProcessImplicitList { if (-d $programname) { next; } # Ignore directories if ($programname eq "runtests") { next; } # Ignore self if (-x $programname) { - $total_count++; + $total_run++; &RunMPIProgram( $programname, $np_default, "", "", "", "", "", "" ); } } @@ -511,7 +539,7 @@ sub ProcessImplicitList { # Skip messages from ls about no files if (! -s $programname) { next; } $programname =~ s/\.c//; - $total_count++; + $total_run++; if (&BuildMPIProgram( $programname ) == 0) { &RunMPIProgram( $programname, $np_default, "", "", "", "", "", "" ); } @@ -943,7 +971,7 @@ sub RunTestPassed { print XMLOUT "<STATUS>pass</STATUS>$newline"; } if ($tapoutput) { - print TAPOUT "ok ${total_count} - $workdir/$programname $np\n"; + print TAPOUT "ok ${total_run} - $workdir/$programname $np\n"; } } sub RunTestFailed { @@ -966,7 +994,7 @@ sub RunTestFailed { } if ($tapoutput) { - print TAPOUT "not ok ${total_count} - $workdir/$programname $np\n"; + print TAPOUT "not ok ${total_run} - $workdir/$programname $np\n"; print TAPOUT " ---\n"; print TAPOUT " Directory: $workdir\n"; print TAPOUT " File: $programname\n"; @@ -996,6 +1024,20 @@ sub RunTestFailed { } } } + +sub SkippedTest { + my $programname = shift; + my $np = shift; + my $workdir = shift; + my $reason = shift; + + # simply omit from the XML output + + if ($tapoutput) { + print TAPOUT "ok ${total_seen} - $workdir/$programname $np # SKIP $reason\n"; + } +} + # ---------------------------------------------------------------------------- # Alternate init routines sub InitQuickTimeout { ----------------------------------------------------------------------- Summary of changes: test/mpi/datatype/testlist | 2 +- test/mpi/perf/testlist | 8 ++-- test/mpi/runtests.in | 130 +++++++++++++++++++++++++++++++++----------- 3 files changed, 103 insertions(+), 37 deletions(-) hooks/post-receive -- MPICH primary repository