December 2013 - commits - mailman3.cels.anl.gov

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-102-gc5229df
by noreply＠mpich.org 19 Dec '13

19 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via c5229dfad83e61e8c32902b95de594cbb696db51 (commit) via ae36d5dbfe5f926e8d2e1619b8720be351813143 (commit) via 80419d6d3b1a658d88886b645654e2297fa28321 (commit) via 35a996524a0a58941829118f7eccac76a6ca3dba (commit) from 61f260051e6244cf3986e8ee1b90a6cc67672b60 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/c5229dfad83e61e8c32902b95de594cbb… commit c5229dfad83e61e8c32902b95de594cbb696db51 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 14:57:47 2013 -0600 Print dummy arg names defined by MPI to f90 files When printing Fortran subroutine prototypes to the f90 module files, use dummy argument (i.e., C formal argument) names defined by MPI, instead of short names like v0, v1. The side-effect is that longer names may make a line exceed the length limit set by Fortran. So we also need to properly wrap long lines. The revise is to support Fortran keyword arguments. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/binding/f90/buildiface b/src/binding/f90/buildiface index c723b15..6580643 100755 --- a/src/binding/f90/buildiface +++ b/src/binding/f90/buildiface @@ -40,6 +40,7 @@ $outfile_prefix = "mpi"; %CtoFName = (); %mpi_routines = (); %NeedConstants = (); # constants needed for declaration, hashed by routine +my $line_limit = 80; # Max line length # # ToDo: Fortran 90 allows some additional checks not possible in Fortran 77. @@ -51,7 +52,7 @@ $outfile_prefix = "mpi"; # # -# parmc2f translates the C/C++ names to the Fortran 90 name. %name% will +# argtypec2f translates the C/C++ names to the Fortran 90 name. %name% will # be replaced with the argument name in declarations. # # Some picky compilers want an interface description where Fortran 77 @@ -60,7 +61,7 @@ $outfile_prefix = "mpi"; # INTERFACE %nl%SUBROUTINE %name%(<args>)%nl%<type decls>%nl%END SUBROUTINE%nl%END INTERFACE # where %nl% is newline/indent. # -%parmc2f = ( 'int' => 'INTEGER', +%argtypec2f = ( 'int' => 'INTEGER', 'int[]' => 'INTEGER %name%(*)', 'int[][3]' => 'INTEGER %name%(3,*)', 'int*' => 'INTEGER', # assume output scalar (see array @@ -120,7 +121,7 @@ $outfile_prefix = "mpi"; 'char*' => 'CHARACTER (LEN=*)', 'char[]' => 'CHARACTER (LEN=*)', 'char*[]' => 'CHARACTER (LEN=*) %name%(*)', - 'char**[]' => 'CHARACTER (LEN=*) %name%(v0,*)', #special case + 'char**[]' => 'CHARACTER (LEN=*) %name%(count,*)', #special case # from Comm_Spawn_multiple 'MPI_Win' => 'INTEGER', 'MPI_Win*' => 'INTEGER', # Never an array of win @@ -387,12 +388,13 @@ foreach $routine (keys(%mpi_routines)) { $routine = $CtoFName{$routine}; } $ucname = uc($routine); - $args = $mpi_routines{$routine}; - @parms = split(/,/, $args ); + my @argtypes = split(/,/, $mpi_routines{$routine}[0]); + my @argnames = split(/,/, $mpi_routines{$routine}[1]); - print "$routine\n" if $gDebug; + print "Trying to bind $routine\n" if $gDebug; # Check for a routine to skip if (defined($skip_routines{$routine})) { + print "Skipping $routine as required\n" if $gDebug; next; } @@ -405,19 +407,20 @@ foreach $routine (keys(%mpi_routines)) { # As noted above, we don't include the routines with choice arguments # in the base module. - if ($args =~ /void/) { - $mpi_choice_routines{$routine} = $args; + if ($mpi_routines{$routine}[0] =~ /void/) { + $mpi_choice_routines{$routine} = $mpi_routines{$routine}[0]; print "Skipping $routine because of void argument\n" if $gDebug; next; } - print MPIBASEFD " SUBROUTINE $out_prefix$ucname("; - for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; - } - print MPIBASEFD "ierror)\n"; + print MPIBASEFD " SUBROUTINE $out_prefix$ucname"; + &PrintArgBrace(MPIBASEFD, + length(" SUBROUTINE $out_prefix$ucname"), + length(" SUBROUTINE "), + $line_limit, + @argnames); &PrintArgDecls( $routine, 0, "" ); - print MPIBASEFD " END SUBROUTINE $out_prefix$ucname\n\n"; + print MPIBASEFD " END SUBROUTINE $out_prefix$ucname\n\n"; } # Add special routines (e.g., the ones with unusual arguments) @@ -455,80 +458,97 @@ if ($is_MPI) { \@WTIME_DOUBLE_TYPE\@ PMPI_WTICK END FUNCTION PMPI_WTICK - SUBROUTINE MPI_NULL_DELETE_FN(a,b,c,d,e) - INTEGER a,b,c,d,e + SUBROUTINE MPI_NULL_DELETE_FN(COMM, KEYVAL, ATTRIBUTE_VAL,& + EXTRA_STATE, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER COMM, KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE END SUBROUTINE MPI_NULL_DELETE_FN - SUBROUTINE MPI_DUP_FN(a,b,c,d,e,f,g) - INTEGER a,b,c,d,e,g - LOGICAL f + SUBROUTINE MPI_DUP_FN(OLDCOMM, KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDCOMM, KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_DUP_FN - SUBROUTINE MPI_NULL_COPY_FN(a,b,c,d,e,f,g) - INTEGER a,b,c,d,e,g - LOGICAL f + SUBROUTINE MPI_NULL_COPY_FN(OLDCOMM, KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDCOMM, KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_NULL_COPY_FN - SUBROUTINE MPI_COMM_NULL_DELETE_FN(a,b,c,d,e) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,e - INTEGER (KIND=MPI_ADDRESS_KIND) c, d + SUBROUTINE MPI_COMM_NULL_DELETE_FN(COMM, COMM_KEYVAL, ATTRIBUTE_VAL,& + EXTRA_STATE, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER COMM, COMM_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE END SUBROUTINE MPI_COMM_NULL_DELETE_FN - SUBROUTINE MPI_COMM_DUP_FN(a,b,c,d,e,f,g) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,g - INTEGER (KIND=MPI_ADDRESS_KIND) c,d,e - LOGICAL f + SUBROUTINE MPI_COMM_DUP_FN(OLDCOMM, COMM_KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDCOMM, COMM_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_COMM_DUP_FN - SUBROUTINE MPI_COMM_NULL_COPY_FN(a,b,c,d,e,f,g) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,g - INTEGER (KIND=MPI_ADDRESS_KIND) c,d,e - LOGICAL f + SUBROUTINE MPI_COMM_NULL_COPY_FN(OLDCOMM, COMM_KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDCOMM, COMM_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_COMM_NULL_COPY_FN - SUBROUTINE MPI_TYPE_NULL_DELETE_FN(a,b,c,d,e) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,e - INTEGER (KIND=MPI_ADDRESS_KIND) c, d + SUBROUTINE MPI_TYPE_NULL_DELETE_FN(DATATYPE, TYPE_KEYVAL, ATTRIBUTE_VAL,& + EXTRA_STATE, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER DATATYPE, TYPE_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE END SUBROUTINE MPI_TYPE_NULL_DELETE_FN - SUBROUTINE MPI_TYPE_DUP_FN(a,b,c,d,e,f,g) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,g - INTEGER (KIND=MPI_ADDRESS_KIND) c,d,e - LOGICAL f + SUBROUTINE MPI_TYPE_DUP_FN(OLDTYPE, TYPE_KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDTYPE, TYPE_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_TYPE_DUP_FN - SUBROUTINE MPI_TYPE_NULL_COPY_FN(a,b,c,d,e,f,g) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,g - INTEGER (KIND=MPI_ADDRESS_KIND) c,d,e - LOGICAL f + SUBROUTINE MPI_TYPE_NULL_COPY_FN(OLDTYPE, TYPE_KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDTYPE, TYPE_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_TYPE_NULL_COPY_FN - SUBROUTINE MPI_WIN_NULL_DELETE_FN(a,b,c,d,e) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,e - INTEGER (KIND=MPI_ADDRESS_KIND) c, d + SUBROUTINE MPI_WIN_NULL_DELETE_FN(WIN, WIN_KEYVAL, ATTRIBUTE_VAL,& + EXTRA_STATE, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER WIN, WIN_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE END SUBROUTINE MPI_WIN_NULL_DELETE_FN - SUBROUTINE MPI_WIN_DUP_FN(a,b,c,d,e,f,g) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,g - INTEGER (KIND=MPI_ADDRESS_KIND) c,d,e - LOGICAL f + SUBROUTINE MPI_WIN_DUP_FN(OLDWIN, WIN_KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDWIN, WIN_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_WIN_DUP_FN - SUBROUTINE MPI_WIN_NULL_COPY_FN(a,b,c,d,e,f,g) - USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND - INTEGER a,b,g - INTEGER (KIND=MPI_ADDRESS_KIND) c,d,e - LOGICAL f + SUBROUTINE MPI_WIN_NULL_COPY_FN(OLDWIN, WIN_KEYVAL, EXTRA_STATE,& + ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR) + USE MPI_CONSTANTS,ONLY: MPI_ADDRESS_KIND + INTEGER OLDWIN, WIN_KEYVAL, IERROR + INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT + LOGICAL FLAG END SUBROUTINE MPI_WIN_NULL_COPY_FN - "; } # Here's where we need to place the interface definitions for the functions @@ -563,44 +583,45 @@ if ($buildScalarVector) { # This is much like the base name (interface) block code foreach my $routine (keys(%scalarVectorRoutines)) { $ucname = uc($routine); - $args = $mpi_routines{$routine}; - @parms = split(/,/, $args ); + my @argtypes = split(/,/, $mpi_routines{$routine}[0]); + my @argnames = split(/,/, $mpi_routines{$routine}[1]); $svArgs = $scalarVectorRoutines{$routine}; # The scalar version - print MPIBASEFD " SUBROUTINE ${out_prefix}${ucname}_S("; - for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; - } - print MPIBASEFD "ierror)\n"; + print MPIBASEFD " SUBROUTINE ${out_prefix}${ucname}_S"; + &PrintArgBrace(MPIBASEFD, + length(" SUBROUTINE ${out_prefix}${ucname}_S"), + length(" SUBROUTINE "), + $line_limit, + @argnames); &PrintArgDecls( $routine, 1, $svArgs ); print MPIBASEFD " EXTERNAL ${out_prefix}${ucname}\n"; - print MPIBASEFD " call ${out_prefix}$ucname("; - for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; - } - print MPIBASEFD "ierror)\n"; - + print MPIBASEFD " call ${out_prefix}$ucname"; + &PrintArgBrace(MPIBASEFD, + length(" call ${out_prefix}$ucname"), + length(" call "), + $line_limit, + @argnames); print MPIBASEFD " END SUBROUTINE ${out_prefix}${ucname}_S\n\n"; # The vector version print MPIBASEFD " SUBROUTINE ${out_prefix}${ucname}_V("; - for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; - } - print MPIBASEFD "ierror)\n"; + &PrintArgBrace(MPIBASEFD, + length(" SUBROUTINE ${out_prefix}${ucname}_S"), + length(" SUBROUTINE "), + $line_limit, + @argnames); &PrintArgDecls( $routine, 0, "" ); print MPIBASEFD " EXTERNAL ${out_prefix}${ucname}\n"; - print MPIBASEFD " call ${out_prefix}$ucname("; - for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; - } - print MPIBASEFD "ierror)\n"; - + print MPIBASEFD " call ${out_prefix}$ucname"; + &PrintArgBrace(MPIBASEFD, + length(" call ${out_prefix}$ucname"), + length(" call "), + $line_limit, + @argnames); print MPIBASEFD " END SUBROUTINE ${out_prefix}${ucname}_V\n\n"; - } } @@ -765,17 +786,18 @@ foreach $routine (keys(%mpi_choice_routines)) { # For each choice routine, add the modules foreach $routine (keys(%mpi_choice_routines)) { $ucname = uc($routine); - $args = $mpi_routines{$routine}; - @parms = split(/,/, $args ); + my @argtypes = split(/,/, $mpi_routines{$routine}[0]); + my @argnames = split(/,/, $mpi_routines{$routine}[1]); - print MPIFD " SUBROUTINE ${out_prefix}${ucname}_T("; - for ($i=0; $i<=$#parms; $i++) { - print MPIFD "v$i,"; - } - print MPIFD "ierror)\n"; + print MPIFD " SUBROUTINE ${out_prefix}${ucname}_T"; + &PrintArgBrace(MPIFD, + length(" SUBROUTINE ${out_prefix}${ucname}_T"), + length(" SUBROUTINE "), + $line_limit, + @argnames); if (defined($NeedConstants{$routine})) { - print MPIFD " USE ${out_prefix}CONSTANTS,ONLY:"; + print MPIFD " USE ${out_prefix}CONSTANTS,ONLY:"; $sep = ""; foreach $name (split(/\s+/,$NeedConstants{$routine})) { print MPIFD "$sep$name"; @@ -788,59 +810,60 @@ foreach $routine (keys(%mpi_choice_routines)) { # convert %type% to the various types and %dims% to the dimensions, # including scalar. $nchoice = 0; - for ($i=0; $i<=$#parms; $i++) { - $parms[$i] =~ s/^const\s//; # Remove const if present - $parm = $parms[$i]; + for ($i=0; $i<=$#argtypes; $i++) { + $argtypes[$i] =~ s/^const\s//; # Remove const if present + $argtype = $argtypes[$i]; # Check for special args $loc = $i+1; if (defined($special_args{"$routine-$loc"})) { - $parm = $special_args{"$routine-$loc"}; + $argtype = $special_args{"$routine-$loc"}; } - if ($parm =~ /void/) { + if ($argtype =~ /void/) { # An alternative to this is to have a separate file for # routines with 2 choice arguments if ($nchoice == 0) { - print MPIFD " <type> v$i<dims>\n"; + print MPIFD " <type> $argnames[$i]<dims>\n"; } else { - print MPIFD " <type$nchoice> v$i<dims$nchoice>\n"; + print MPIFD " <type$nchoice> $argnames[$i]<dims$nchoice>\n"; } $nchoice ++; } else { # Map the C type to the Fortran type - $cparm = $parm; - $cparm =~ s/\s+//g; - $fparm = $parmc2f{$cparm}; - if ($fparm eq "") { - print STDERR "$routine: No parm type for $cparm ($parm)\n"; + $cargtype = $argtype; + $cargtype =~ s/\s+//g; + $fargtype = $argtypec2f{$cargtype}; + if ($fargtype eq "") { + print STDERR "$routine: No Fortran type for $cargtype ($argtype)\n"; } - if ($fparm =~ /%name%/) { - $fparm =~ s/%name%/v$i/; + if ($fargtype =~ /%name%/) { + $fargtype =~ s/%name%/$argnames[$i]/; # In the name case, convert any %nl% to newlines and spaces - $fparm =~ s/%nl%/\n /g; - print MPIFD " $fparm\n"; + $fargtype =~ s/%nl%/\n /g; + print MPIFD " $fargtype\n"; } else { - print MPIFD " $fparm v$i\n"; + print MPIFD " $fargtype $argnames[$i]\n"; } } } - print MPIFD " INTEGER ierror\n"; print MPIFD " EXTERNAL ${out_prefix}${ucname}\n"; - print MPIFD " CALL ${out_prefix}${ucname}("; - for ($i=0; $i<=$#parms; $i++) { - print MPIFD "v$i,"; - } - print MPIFD "ierror)\n"; - print MPIFD " END SUBROUTINE ${out_prefix}${ucname}_T\n\n"; + print MPIFD " CALL ${out_prefix}${ucname}"; + &PrintArgBrace(MPIFD, + length(" CALL ${out_prefix}${ucname}"), + length(" CALL "), + $line_limit, + @argnames); + + print MPIFD " END SUBROUTINE ${out_prefix}${ucname}_T\n\n"; } # The base sizeof's are handled separately now in their own file -print MPIFD " END MODULE ${ucoutfile_prefix}_t1_s\n"; +print MPIFD " END MODULE ${ucoutfile_prefix}_t1_s\n"; close MPIFD; &ReplaceIfDifferent( "${outfile_prefix}_t1.f90", "${outfile_prefix}_t1.f90.new" ); @@ -1220,13 +1243,49 @@ EOT } +# Print arguments in a pair of braces and end with a new line. +# $fd : File handle to print to. +# $curlen : Length of the current line. +# $indent : Indention length of the following lines when they exist. +# $line_length : Maximal length of a line. +# @argnames : argument names to print out. +sub PrintArgBrace { + my ($fd, $curlen, $indent, $line_length, @argnames) = @_; + my $i = 0; + + print $fd "("; + $curlen++; + foreach my $arg (@argnames) { + if ($curlen + length($arg) + 2 > $line_length) { # + 2 for the trailing ',&' + $curlen = $indent; + print $fd "&\n"; + print $fd " " x $indent; + print $fd "$arg"; + } else { + print $fd "$arg"; + } + + if ($i++ < $#argnames) { + print $fd ","; + } + + $curlen += length($arg) + 1; + } + + print $fd ")\n"; +} + # Print the declarations for the given routine. sub PrintArgDecls { my ($routine,$svflag,$svArgs) = @_; my $ucname = uc($routine); - my $args = $mpi_routines{$routine}; - my @parms = split(/,/, $args ); + my @argtypes = split(/,/, $mpi_routines{$routine}[0]); + my @argnames = split(/,/, $mpi_routines{$routine}[1]); + + print "Printing argument delartion for $ucname\n" if $gDebug; + print "argtypes = $mpi_routines{$routine}[0]\n" if $gDebug; + print "argnames = $mpi_routines{$routine}[1]\n" if $gDebug; # preload the svargs if requested. This is used to decide whether # an array arg is output as a scalar or a vector @@ -1246,29 +1305,29 @@ sub PrintArgDecls { # MPI_OFFSET_KIND) my %use_constants = (); my $found_constants = 0; - for (my $i=0; $i<=$#parms; $i++) { - $parms[$i] =~ s/^const\s+//; # Remove const if present - $parm = $parms[$i]; + for (my $i=0; $i<=$#argtypes; $i++) { + $argtypes[$i] =~ s/^const\s+//; # Remove const if present + my $argtype = $argtypes[$i]; # Check for special args $loc = $i+1; if (defined($special_args{"$routine-$loc"})) { - $parm = $special_args{"$routine-$loc"}; + $argtype = $special_args{"$routine-$loc"}; } # Map the C type to the Fortran type - $cparm = $parm; - $cparm =~ s/\s+//g; - $fparm = $parmc2f{$cparm}; + my $cargtype = $argtype; + $cargtype =~ s/\s+//g; + my $fargtype = $argtypec2f{$cargtype}; # Now, does this type contain an MPI constant? - if (!defined($fparm)) { - print "$cparm value has no matching fortran parm\n"; + if (!defined($fargtype)) { + print "$cargtype value has no matching fortran argtype\n"; } - if ($fparm =~ /(MPI_[A-Z_]*)/) { + if ($fargtype =~ /(MPI_[A-Z_]*)/) { $use_constants{$1} = 1; $found_constants = 1; } } if ($found_constants) { - print MPIBASEFD " USE MPI_CONSTANTS,ONLY:"; + print MPIBASEFD " USE MPI_CONSTANTS,ONLY:"; $sep = ""; foreach $name (keys(%use_constants)) { print MPIBASEFD "$sep$name"; @@ -1279,30 +1338,29 @@ sub PrintArgDecls { } # Output argument types - $lastParmType = "<none>"; - for (my $i=0; $i<=$#parms; $i++) { - $parm = $parms[$i]; + for (my $i=0; $i<=$#argtypes; $i++) { + $argtype = $argtypes[$i]; # Check for special args $loc = $i+1; if (defined($special_args{"$routine-$loc"})) { - $parm = $special_args{"$routine-$loc"}; + $argtype = $special_args{"$routine-$loc"}; } # Map the C type to the Fortran type - $cparm = $parm; - $cparm =~ s/\s+//g; - $fparm = $parmc2f{$cparm}; - if ($fparm eq "") { - print STDERR "$routine: No parm type for $cparm ($parm)\n"; + $cargtype = $argtype; + $cargtype =~ s/\s+//g; + $fargtype = $argtypec2f{$cargtype}; + if ($fargtype eq "") { + print STDERR "$routine: No Fortran type for $cargtype ($argtype)\n"; } # Split out the base type from the name - if ($fparm =~ /(\w+.*)\s+(%name\S.*)/) { - $parmType = $1; + if ($fargtype =~ /(\w+.*)\s+(%name\S.*)/) { + $varType = $1; $varName = $2; if ($varName =~ /%name%/) { - $varName =~ s/%name%/v$i/; + $varName =~ s/%name%/$argnames[$i]/; } $varName =~ s/%nl%/\n /g; - $parmType =~ s/%nl%/\n /g; + $varType =~ s/%nl%/\n /g; # Here's where we might change vector to scalar args if ($svflag) { @@ -1317,26 +1375,17 @@ sub PrintArgDecls { } else { print STDERR "Failed to make arg $i in $routine a scalar\n"; + } } } - } - } - else { - $parmType = $fparm; - $varName = "v$i"; - } - if ($parmType ne $lastParmType) { - if ($lastParmType ne "<none>" ) { print MPIBASEFD "\n"; } - print MPIBASEFD " $parmType $varName"; - $lastParmType = $parmType; - } - else { - print MPIBASEFD ", $varName"; - } - } - if ($lastParmType ne "<none>" ) { print MPIBASEFD "\n"; } + } + else { + $varType = $fargtype; + $varName = $argnames[$i]; + } - print MPIBASEFD " INTEGER ierror\n"; + print MPIBASEFD " $varType $varName\n"; + } } # http://git.mpich.org/mpich.git/commitdiff/ae36d5dbfe5f926e8d2e1619b8720be35… commit ae36d5dbfe5f926e8d2e1619b8720be351813143 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 14:18:14 2013 -0600 Add indentions, remove white spaces, replace Tabs Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/binding/f90/buildiface b/src/binding/f90/buildiface index 8965956..c723b15 100755 --- a/src/binding/f90/buildiface +++ b/src/binding/f90/buildiface @@ -2,14 +2,14 @@ # FIXME: # TODO: -# For MPI-3 (See 17.1.3 for details), need to add (for mpi module; mpi_f08 +# For MPI-3 (See 17.1.3 for details), need to add (for mpi module; mpi_f08 # module will require different system entirely): # *ALL* functions declared # This will require the use of non-standard, compiler-specific options, # and may require overriding user choices about warning levels and options # in order to handle the choice arguments # Provide DERIVED TYPES for MPI_STATUS and all handles (as in the mpi_f08 -# module). For all handle types, provide overloads for .EQ., .NE., == +# module). For all handle types, provide overloads for .EQ., .NE., == # and /= operators (Partially done) # Check on MPI_F_SYNC_REG, mentioned on page 598. @@ -58,206 +58,206 @@ $outfile_prefix = "mpi"; # was happy with a simple EXTERNAL. To handle this, the EXTERNAL # has a more elaborate form: # INTERFACE %nl%SUBROUTINE %name%(<args>)%nl%<type decls>%nl%END SUBROUTINE%nl%END INTERFACE -# where %nl% is newline/indent. +# where %nl% is newline/indent. # %parmc2f = ( 'int' => 'INTEGER', - 'int[]' => 'INTEGER %name%(*)', - 'int[][3]' => 'INTEGER %name%(3,*)', - 'int*' => 'INTEGER', # assume output scalar (see array - # replacement below) - 'bool' => 'LOGICAL', - 'bool[]' => 'LOGICAL %name%(*)', - 'MPI_Handler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', - 'MPI_Win_errhandler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', - 'MPI_Comm_errhandler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', - 'MPI_File_errhandler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', - # These other functions have <choice> (really void*) arguments - # and so an interface spec is very hard to do in Fortran 90. - 'MPI_Comm_copy_attr_function*' => 'EXTERNAL', - 'MPI_Comm_delete_attr_function*' => 'EXTERNAL', - 'MPI_Type_copy_attr_function*' => 'EXTERNAL', - 'MPI_Type_delete_attr_function*' => 'EXTERNAL', - 'MPI_Win_copy_attr_function*' => 'EXTERNAL', - 'MPI_Win_delete_attr_function*' => 'EXTERNAL', - 'MPI_Copy_function*' => 'EXTERNAL', - 'MPI_Delete_function*' => 'EXTERNAL', - 'MPI_User_function*' => 'EXTERNAL', - 'MPI_Grequest_query_function*' => 'EXTERNAL', - 'MPI_Grequest_free_function*' => 'EXTERNAL', - 'MPI_Grequest_cancel_function*' => 'EXTERNAL', - 'MPI_Datarep_conversion_function*' => 'EXTERNAL', - 'MPI_Datarep_extent_function*' => 'EXTERNAL', - 'MPI_Request' => 'INTEGER', - 'MPI_Request*' => 'INTEGER', - 'MPI_Request[]' => 'INTEGER %name%(*)', - 'MPIO_Request' => 'INTEGER', - 'MPIO_Request*' => 'INTEGER', - 'MPI_Datatype' => 'INTEGER', - 'MPI_Datatype*' => 'INTEGER', - 'MPI_Datatype[]' => 'INTEGER %name%(*)', - 'MPI_Comm' => 'INTEGER', - 'MPI_Comm*' => 'INTEGER', # Never an array of comm - 'MPI_Group' => 'INTEGER', - 'MPI_Group*' => 'INTEGER', # Never an array of groups - 'MPI_Errhandler' => 'INTEGER', - 'MPI_Errhandler*' => 'INTEGER', # Never an array of errhandlers - 'MPI_Op' => 'INTEGER', - 'MPI_Op*' => 'INTEGER', # Never an array of ops - 'MPI_Message' => 'INTEGER', - 'MPI_Message*' => 'INTEGER', # Never an array of messages - 'MPI_Status*' => 'INTEGER %name%(MPI_STATUS_SIZE)', - 'MPI_Status[]' => 'INTEGER %name%(MPI_STATUS_SIZE,*)', - 'MPI_Aint' => 'INTEGER(KIND=MPI_ADDRESS_KIND)', - 'MPI_Aint*' => 'INTEGER(KIND=MPI_ADDRESS_KIND)', - 'MPI_Aint[]' => 'INTEGER(KIND=MPI_ADDRESS_KIND) %name%(*)', - 'MPI_Count' => 'INTEGER(KIND=MPI_COUNT_KIND)', - 'MPI_Count*' => 'INTEGER(KIND=MPI_COUNT_KIND)', - 'MPI_Offset' => 'INTEGER(KIND=MPI_OFFSET_KIND)', - 'MPI_Offset*' => 'INTEGER(KIND=MPI_OFFSET_KIND)', - 'MPI_Info' => 'INTEGER', - 'MPI_Info*' => 'INTEGER', # Never an array of info - 'MPI_Info[]' => 'INTEGER %name%(*)', - 'char*' => 'CHARACTER (LEN=*)', - 'char[]' => 'CHARACTER (LEN=*)', - 'char*[]' => 'CHARACTER (LEN=*) %name%(*)', - 'char**[]' => 'CHARACTER (LEN=*) %name%(v0,*)', #special case - # from Comm_Spawn_multiple - 'MPI_Win' => 'INTEGER', - 'MPI_Win*' => 'INTEGER', # Never an array of win - 'MPI_File' => 'INTEGER', - 'MPI_File*' => 'INTEGER', # Never an array of files - 'MPI_Message' => 'INTEGER', - 'MPI_Message*' => 'INTEGER', # Never an array of messages - ); + 'int[]' => 'INTEGER %name%(*)', + 'int[][3]' => 'INTEGER %name%(3,*)', + 'int*' => 'INTEGER', # assume output scalar (see array + # replacement below) + 'bool' => 'LOGICAL', + 'bool[]' => 'LOGICAL %name%(*)', + 'MPI_Handler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', + 'MPI_Win_errhandler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', + 'MPI_Comm_errhandler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', + 'MPI_File_errhandler_function*' => 'INTERFACE %nl%SUBROUTINE %name%(vv0,vv1)%nl%INTEGER vv0,vv1%nl%END SUBROUTINE%nl%END INTERFACE', + # These other functions have <choice> (really void*) arguments + # and so an interface spec is very hard to do in Fortran 90. + 'MPI_Comm_copy_attr_function*' => 'EXTERNAL', + 'MPI_Comm_delete_attr_function*' => 'EXTERNAL', + 'MPI_Type_copy_attr_function*' => 'EXTERNAL', + 'MPI_Type_delete_attr_function*' => 'EXTERNAL', + 'MPI_Win_copy_attr_function*' => 'EXTERNAL', + 'MPI_Win_delete_attr_function*' => 'EXTERNAL', + 'MPI_Copy_function*' => 'EXTERNAL', + 'MPI_Delete_function*' => 'EXTERNAL', + 'MPI_User_function*' => 'EXTERNAL', + 'MPI_Grequest_query_function*' => 'EXTERNAL', + 'MPI_Grequest_free_function*' => 'EXTERNAL', + 'MPI_Grequest_cancel_function*' => 'EXTERNAL', + 'MPI_Datarep_conversion_function*' => 'EXTERNAL', + 'MPI_Datarep_extent_function*' => 'EXTERNAL', + 'MPI_Request' => 'INTEGER', + 'MPI_Request*' => 'INTEGER', + 'MPI_Request[]' => 'INTEGER %name%(*)', + 'MPIO_Request' => 'INTEGER', + 'MPIO_Request*' => 'INTEGER', + 'MPI_Datatype' => 'INTEGER', + 'MPI_Datatype*' => 'INTEGER', + 'MPI_Datatype[]' => 'INTEGER %name%(*)', + 'MPI_Comm' => 'INTEGER', + 'MPI_Comm*' => 'INTEGER', # Never an array of comm + 'MPI_Group' => 'INTEGER', + 'MPI_Group*' => 'INTEGER', # Never an array of groups + 'MPI_Errhandler' => 'INTEGER', + 'MPI_Errhandler*' => 'INTEGER', # Never an array of errhandlers + 'MPI_Op' => 'INTEGER', + 'MPI_Op*' => 'INTEGER', # Never an array of ops + 'MPI_Message' => 'INTEGER', + 'MPI_Message*' => 'INTEGER', # Never an array of messages + 'MPI_Status*' => 'INTEGER %name%(MPI_STATUS_SIZE)', + 'MPI_Status[]' => 'INTEGER %name%(MPI_STATUS_SIZE,*)', + 'MPI_Aint' => 'INTEGER(KIND=MPI_ADDRESS_KIND)', + 'MPI_Aint*' => 'INTEGER(KIND=MPI_ADDRESS_KIND)', + 'MPI_Aint[]' => 'INTEGER(KIND=MPI_ADDRESS_KIND) %name%(*)', + 'MPI_Count' => 'INTEGER(KIND=MPI_COUNT_KIND)', + 'MPI_Count*' => 'INTEGER(KIND=MPI_COUNT_KIND)', + 'MPI_Offset' => 'INTEGER(KIND=MPI_OFFSET_KIND)', + 'MPI_Offset*' => 'INTEGER(KIND=MPI_OFFSET_KIND)', + 'MPI_Info' => 'INTEGER', + 'MPI_Info*' => 'INTEGER', # Never an array of info + 'MPI_Info[]' => 'INTEGER %name%(*)', + 'char*' => 'CHARACTER (LEN=*)', + 'char[]' => 'CHARACTER (LEN=*)', + 'char*[]' => 'CHARACTER (LEN=*) %name%(*)', + 'char**[]' => 'CHARACTER (LEN=*) %name%(v0,*)', #special case + # from Comm_Spawn_multiple + 'MPI_Win' => 'INTEGER', + 'MPI_Win*' => 'INTEGER', # Never an array of win + 'MPI_File' => 'INTEGER', + 'MPI_File*' => 'INTEGER', # Never an array of files + 'MPI_Message' => 'INTEGER', + 'MPI_Message*' => 'INTEGER', # Never an array of messages + ); # special_args provides for handling of arguments that require special -# features. The keys are of the form 'Routine-count', with count the +# features. The keys are of the form 'Routine-count', with count the # position of the argument, starting from one. %special_args = ( 'Testany-2' => 'MPI_Request[]', - 'Testany-4' => 'bool', - 'Startall-2' => 'MPI_Request[]', - 'Testall-2' => 'MPI_Request[]', - 'Testall-3' => 'bool', - 'Testall-4' => 'MPI_Status[]', - 'Testsome-2' => 'MPI_Request[]', - 'Testsome-4' => 'int[]', - 'Testsome-5' => 'MPI_Status[]', - 'Test-2' => 'bool', - 'Test_cancelled-2' => 'bool', - 'Type_hindexed-2' => 'int[]', - 'Type_hindexed-3' => 'int[]', - 'Type_indexed-2' => 'int[]', - 'Type_indexed-3' => 'int[]', - 'Type_hvector-3' => 'int', - 'Type_struct-2' => 'int[]', - 'Type_struct-3' => 'int[]', - 'Type_struct-4' => 'MPI_Datatype[]', - 'Type_extent-2' => 'int', - 'Type_lb-2' => 'int', - 'Type_ub-2' => 'int', - 'Waitall-2' => 'MPI_Request[]', - 'Waitall-3' => 'MPI_Status[]', - 'Waitany-2' => 'MPI_Request[]', - 'Waitsome-2' => 'MPI_Request[]', - 'Waitsome-4' => 'int[]', - 'Waitsome-5' => 'MPI_Status[]', - 'Group_excl-3' => 'int[]', - 'Group_incl-3' => 'int[]', - 'Group_translate_ranks-3' => 'int[]', - 'Group_translate_ranks-5' => 'int[]', - 'Cart_coords-4' => 'int[]', - 'Cart_create-3' => 'int[]', - 'Cart_create-4' => 'bool[]', - 'Cart_get-3' => 'int[]', - 'Cart_get-5' => 'int[]', - 'Cart_get-4' => 'bool[]', - 'Cart_map-3' => 'int[]', - 'Cart_map-4' => 'bool[]', - 'Cart_rank-2' => 'int[]', - 'Cart_sub-2' => 'bool[]', - 'Dims_create-3' => 'int[]', - 'Graph_create-3' => 'int[]', - 'Graph_create-4' => 'int[]', - 'Graph_create-5' => 'bool', - 'Graph_get-4' => 'int[]', - 'Graph_get-5' => 'int[]', - 'Graph_map-3' => 'int[]', - 'Graph_map-4' => 'int[]', - 'Graph_neighbors-4' => 'int[]', - 'Dist_graph_create-8' => 'bool', - 'Dist_graph_create_adjacent-9' => 'bool', - 'Dist_graph_neighbors_count-4' => 'bool', - 'Allgatherv-5' => 'int[]', - 'Allgatherv-6' => 'int[]', - 'Alltoallv-2' => 'int[]', - 'Alltoallv-3' => 'int[]', - 'Alltoallv-6' => 'int[]', - 'Alltoallv-7' => 'int[]', - 'Alltoallw-2' => 'int[]', - 'Alltoallw-3' => 'int[]', - 'Alltoallw-6' => 'int[]', - 'Alltoallw-7' => 'int[]', - 'Gatherv-5' => 'int[]', - 'Gatherv-6' => 'int[]', - 'Iallgatherv-5' => 'int[]', - 'Iallgatherv-6' => 'int[]', - 'Ialltoallv-2' => 'int[]', - 'Ialltoallv-3' => 'int[]', - 'Ialltoallv-6' => 'int[]', - 'Ialltoallv-7' => 'int[]', - 'Ialltoallw-2' => 'int[]', - 'Ialltoallw-3' => 'int[]', - 'Ialltoallw-6' => 'int[]', - 'Ialltoallw-7' => 'int[]', - 'Igatherv-5' => 'int[]', - 'Igatherv-6' => 'int[]', - 'Ireduce_scatter-3' => 'int[]', - 'Iscatterv-2' => 'int[]', - 'Iscatterv-3' => 'int[]', - 'Reduce_scatter-3' => 'int[]', - 'Scatterv-2' => 'int[]', - 'Scatterv-3' => 'int[]', - 'Iprobe-4' => 'bool', - 'Improbe-4' => 'bool', - 'Op_create-2' => 'bool', - 'Attr_get-4' => 'bool', - 'Comm_get_attr-4' => 'bool', - 'Type_get_attr-4' => 'bool', - 'Win_get_attr-4' => 'bool', - 'Comm_test_inter-2' => 'bool', - 'Intercomm_merge-2' => 'bool', - 'Cart_create-5' => 'bool', - 'Initialized-1' => 'bool', - 'Finalized-1' => 'bool', - 'Group_range_excl-3' => 'int[][3]', - 'Group_range_incl-3' => 'int[][3]', - 'Info_get_valuelen-4' => 'bool', - 'Is_thread_main-1' => 'bool', - 'Type_create_subarray-2' => 'int[]', - 'Type_create_subarray-3' => 'int[]', - 'Type_create_subarray-4' => 'int[]', - 'Request_get_status-2' => 'bool', - 'Status_set_cancelled-2' => 'bool', - 'Info_get-5' => 'bool', - 'Type_create_indexed_block-3' => 'int[]', - 'Type_create_darray-4' => 'int[]', - 'Type_create_darray-5' => 'int[]', - 'Type_create_darray-6' => 'int[]', - 'Type_create_darray-7' => 'int[]', - 'Type_create_struct-2' => 'int[]', - 'Type_create_struct-3' => 'MPI_Aint[]', - 'Win_test-2' => 'bool', - 'Type_create_hindexed-2' => 'int[]', - 'Type_create_hindexed-3' => 'MPI_Aint[]', - 'Op_commutative-2' => 'bool', - 'File_set_atomicity-2' => 'bool', - 'File_get_atomicity-2' => 'bool', - ); + 'Testany-4' => 'bool', + 'Startall-2' => 'MPI_Request[]', + 'Testall-2' => 'MPI_Request[]', + 'Testall-3' => 'bool', + 'Testall-4' => 'MPI_Status[]', + 'Testsome-2' => 'MPI_Request[]', + 'Testsome-4' => 'int[]', + 'Testsome-5' => 'MPI_Status[]', + 'Test-2' => 'bool', + 'Test_cancelled-2' => 'bool', + 'Type_hindexed-2' => 'int[]', + 'Type_hindexed-3' => 'int[]', + 'Type_indexed-2' => 'int[]', + 'Type_indexed-3' => 'int[]', + 'Type_hvector-3' => 'int', + 'Type_struct-2' => 'int[]', + 'Type_struct-3' => 'int[]', + 'Type_struct-4' => 'MPI_Datatype[]', + 'Type_extent-2' => 'int', + 'Type_lb-2' => 'int', + 'Type_ub-2' => 'int', + 'Waitall-2' => 'MPI_Request[]', + 'Waitall-3' => 'MPI_Status[]', + 'Waitany-2' => 'MPI_Request[]', + 'Waitsome-2' => 'MPI_Request[]', + 'Waitsome-4' => 'int[]', + 'Waitsome-5' => 'MPI_Status[]', + 'Group_excl-3' => 'int[]', + 'Group_incl-3' => 'int[]', + 'Group_translate_ranks-3' => 'int[]', + 'Group_translate_ranks-5' => 'int[]', + 'Cart_coords-4' => 'int[]', + 'Cart_create-3' => 'int[]', + 'Cart_create-4' => 'bool[]', + 'Cart_get-3' => 'int[]', + 'Cart_get-5' => 'int[]', + 'Cart_get-4' => 'bool[]', + 'Cart_map-3' => 'int[]', + 'Cart_map-4' => 'bool[]', + 'Cart_rank-2' => 'int[]', + 'Cart_sub-2' => 'bool[]', + 'Dims_create-3' => 'int[]', + 'Graph_create-3' => 'int[]', + 'Graph_create-4' => 'int[]', + 'Graph_create-5' => 'bool', + 'Graph_get-4' => 'int[]', + 'Graph_get-5' => 'int[]', + 'Graph_map-3' => 'int[]', + 'Graph_map-4' => 'int[]', + 'Graph_neighbors-4' => 'int[]', + 'Dist_graph_create-8' => 'bool', + 'Dist_graph_create_adjacent-9' => 'bool', + 'Dist_graph_neighbors_count-4' => 'bool', + 'Allgatherv-5' => 'int[]', + 'Allgatherv-6' => 'int[]', + 'Alltoallv-2' => 'int[]', + 'Alltoallv-3' => 'int[]', + 'Alltoallv-6' => 'int[]', + 'Alltoallv-7' => 'int[]', + 'Alltoallw-2' => 'int[]', + 'Alltoallw-3' => 'int[]', + 'Alltoallw-6' => 'int[]', + 'Alltoallw-7' => 'int[]', + 'Gatherv-5' => 'int[]', + 'Gatherv-6' => 'int[]', + 'Iallgatherv-5' => 'int[]', + 'Iallgatherv-6' => 'int[]', + 'Ialltoallv-2' => 'int[]', + 'Ialltoallv-3' => 'int[]', + 'Ialltoallv-6' => 'int[]', + 'Ialltoallv-7' => 'int[]', + 'Ialltoallw-2' => 'int[]', + 'Ialltoallw-3' => 'int[]', + 'Ialltoallw-6' => 'int[]', + 'Ialltoallw-7' => 'int[]', + 'Igatherv-5' => 'int[]', + 'Igatherv-6' => 'int[]', + 'Ireduce_scatter-3' => 'int[]', + 'Iscatterv-2' => 'int[]', + 'Iscatterv-3' => 'int[]', + 'Reduce_scatter-3' => 'int[]', + 'Scatterv-2' => 'int[]', + 'Scatterv-3' => 'int[]', + 'Iprobe-4' => 'bool', + 'Improbe-4' => 'bool', + 'Op_create-2' => 'bool', + 'Attr_get-4' => 'bool', + 'Comm_get_attr-4' => 'bool', + 'Type_get_attr-4' => 'bool', + 'Win_get_attr-4' => 'bool', + 'Comm_test_inter-2' => 'bool', + 'Intercomm_merge-2' => 'bool', + 'Cart_create-5' => 'bool', + 'Initialized-1' => 'bool', + 'Finalized-1' => 'bool', + 'Group_range_excl-3' => 'int[][3]', + 'Group_range_incl-3' => 'int[][3]', + 'Info_get_valuelen-4' => 'bool', + 'Is_thread_main-1' => 'bool', + 'Type_create_subarray-2' => 'int[]', + 'Type_create_subarray-3' => 'int[]', + 'Type_create_subarray-4' => 'int[]', + 'Request_get_status-2' => 'bool', + 'Status_set_cancelled-2' => 'bool', + 'Info_get-5' => 'bool', + 'Type_create_indexed_block-3' => 'int[]', + 'Type_create_darray-4' => 'int[]', + 'Type_create_darray-5' => 'int[]', + 'Type_create_darray-6' => 'int[]', + 'Type_create_darray-7' => 'int[]', + 'Type_create_struct-2' => 'int[]', + 'Type_create_struct-3' => 'MPI_Aint[]', + 'Win_test-2' => 'bool', + 'Type_create_hindexed-2' => 'int[]', + 'Type_create_hindexed-3' => 'MPI_Aint[]', + 'Op_commutative-2' => 'bool', + 'File_set_atomicity-2' => 'bool', + 'File_get_atomicity-2' => 'bool', + ); # Some routines must be skipped (custom code is provided for them) %skip_routines = ( 'Init' => 1, 'Init_thread' => 1, 'Status_c2f' => 1, - 'Status_f2c' => 1, 'Pcontrol' => 1, - ); + 'Status_f2c' => 1, 'Pcontrol' => 1, + ); # Some routines *may* be skipped if we don't want to handle the possibility # of a scalar or vector argument @@ -265,23 +265,23 @@ $outfile_prefix = "mpi"; # For each of these, we need to know which arguments are the "scalar/vector" # The value of the hash gives us the answer, indexed from 1 # (these are not correct yet). -%scalarVectorRoutines = ( 'Startall' => '2-1', 'Testall' => '2-1:4-1', - 'Testany' => '2-1', - 'Testsome' => '2-1:4-1:5-1', - 'Waitall' => '2-1:3-1', 'Waitany' => '2-1', - 'Waitsome' => '2-1:4-1:5-1', - 'Dims_create' => '3-2', - 'Cart_rank' => '2', 'Cart_coords' => '4-3', - 'Cart_get' => '3-2:4-2:5-2', - 'Graph_neighbors' => '4-3', - 'Cart_sub' => '2', - 'Cart_map' => '3-2:4-2', - 'Cart_create' => '3-2:4-2', - 'Graph_create' => '3:4', - 'Dist_graph_create' => '6', - 'Dist_graph_create_adjacent' => '4:7', - 'Dist_graph_neighbors' => '4:7', - 'Group_translate_ranks' => '3-2:5-2', +%scalarVectorRoutines = ( 'Startall' => '2-1', 'Testall' => '2-1:4-1', + 'Testany' => '2-1', + 'Testsome' => '2-1:4-1:5-1', + 'Waitall' => '2-1:3-1', 'Waitany' => '2-1', + 'Waitsome' => '2-1:4-1:5-1', + 'Dims_create' => '3-2', + 'Cart_rank' => '2', 'Cart_coords' => '4-3', + 'Cart_get' => '3-2:4-2:5-2', + 'Graph_neighbors' => '4-3', + 'Cart_sub' => '2', + 'Cart_map' => '3-2:4-2', + 'Cart_create' => '3-2:4-2', + 'Graph_create' => '3:4', + 'Dist_graph_create' => '6', + 'Dist_graph_create_adjacent' => '4:7', + 'Dist_graph_neighbors' => '4:7', + 'Group_translate_ranks' => '3-2:5-2', ); @@ -292,26 +292,26 @@ $build_io = 1; # Process any options foreach $_ (@ARGV) { if (/--?prototype=(.*)/) { - $prototype_file = $1; + $prototype_file = $1; } elsif (/--?sv/) { - # This obscure argument enables the creation of an interface that - # includes the routines that can accept a scalar or a vector - # (e.g., a single request or an array of requests) on a single - # type (e.g., an integer). By default, we leave these out. - $buildScalarVector = 1; + # This obscure argument enables the creation of an interface that + # includes the routines that can accept a scalar or a vector + # (e.g., a single request or an array of requests) on a single + # type (e.g., an integer). By default, we leave these out. + $buildScalarVector = 1; } elsif (/deffile=(.*)/) { - $definition_file = $1; - $is_MPI = 0; + $definition_file = $1; + $is_MPI = 0; } elsif (/--?noio/) { - $build_io = 0; + $build_io = 0; } elsif (/--?debug/) { $gDebug = 1; } else { - print STDERR "Unrecognized argument $_\n"; - exit 2; + print STDERR "Unrecognized argument $_\n"; + exit 2; } } @@ -323,15 +323,15 @@ if ($definition_file) { $ucoutfile_prefix = uc( $outfile_prefix ); # -# Read the interface file (e.g., mpi.h.in) and file in the various +# Read the interface file (e.g., mpi.h.in) and file in the various # data structures (they're in global variables) &ReadInterface( $prototype_file, $routine_prefix, $routine_pattern, "mpi_routines" ); -if ( -s "../../mpi/romio/include/mpio.h.in" && $build_io) { -# %skipBlocks = ( 'HAVE_MPI_DARRAY_SUBARRAY' => 1, -# 'HAVE_MPI_INFO' => 1, -# 'MPICH' => 1 ); +if ( -s "../../mpi/romio/include/mpio.h.in" && $build_io) { +# %skipBlocks = ( 'HAVE_MPI_DARRAY_SUBARRAY' => 1, +# 'HAVE_MPI_INFO' => 1, +# 'MPICH' => 1 ); &ReadInterface( "../../mpi/romio/include/mpio.h.in", $routine_prefix, - $routine_pattern, "mpi_routines" ); + $routine_pattern, "mpi_routines" ); # %skipBlocks = (); } if ( $buildMPIX ) { @@ -339,9 +339,9 @@ if ( $buildMPIX ) { } # -# For some MPI routines, we need to distinguish between arguments that are +# For some MPI routines, we need to distinguish between arguments that are # input arrays versus ones that are output scalars. For those functions, -# convert input (or output) arrays to [] format. +# convert input (or output) arrays to [] format. # ---------------------------------------------------------------------------- # @@ -350,7 +350,7 @@ if ( $buildMPIX ) { # Print header open (MPIFD, ">${outfile_prefix}.f90.new" ) || die "Could not open ${outfile_prefix}.f90.new\n"; -# Was +# Was # USE MPI_CONSTANTS, & # & BASE_MPI_WTIME => MPI_WTIME, BASE_MPI_WTICK => MPI_WTICK # but this caused problems with the pg compiler. Need to understand and fix @@ -363,28 +363,28 @@ print MPIFD "! -*- Mode: Fortran; -*- USE ${ucoutfile_prefix}_SIZEOFS USE ${ucoutfile_prefix}_BASE END MODULE $ucoutfile_prefix\n"; - + close (MPIFD); &ReplaceIfDifferent( "${outfile_prefix}.f90", "${outfile_prefix}.f90.new" ); # ---------------------------------------------------------------------------- # This is the file for the routines that have no "choice" arguments. -# An example of a choice argument is a "void *buf" input argument to +# An example of a choice argument is a "void *buf" input argument to # MPI_Send, which allows any buffer address, both numeric and character. open ( MPIBASEFD, ">${outfile_prefix}_base.f90.in.new" ) || die "Could not open ${outfile_prefix}_base.f90.in.new\n"; print MPIBASEFD "! -*- Mode: Fortran; -*- ! (C) 2008 by Argonne National Laboratory. -! See COPYRIGHT in top-level directory. +! See COPYRIGHT in top-level directory. MODULE ${ucoutfile_prefix}_BASE IMPLICIT NONE ! This module was created by the script buildiface INTERFACE\n"; foreach $routine (keys(%mpi_routines)) { - # Permit each package to define a new name for the Fortran version of the + # Permit each package to define a new name for the Fortran version of the # routine if (defined($CtoFName{$routine})) { - $routine = $CtoFName{$routine}; + $routine = $CtoFName{$routine}; } $ucname = uc($routine); $args = $mpi_routines{$routine}; @@ -393,25 +393,27 @@ foreach $routine (keys(%mpi_routines)) { print "$routine\n" if $gDebug; # Check for a routine to skip if (defined($skip_routines{$routine})) { - next; + next; } if (defined($scalarVectorRoutines{$routine})) { - # These require special processing in any case - next; + # These require special processing in any case + next; } # Check for a void * argument (usually choice) # As noted above, we don't include the routines with choice arguments # in the base module. + if ($args =~ /void/) { - $mpi_choice_routines{$routine} = $args; - print "Skipping $routine because of void argument\n" if $gDebug; - next; + $mpi_choice_routines{$routine} = $args; + print "Skipping $routine because of void argument\n" if $gDebug; + next; } + print MPIBASEFD " SUBROUTINE $out_prefix$ucname("; for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; + print MPIBASEFD "v$i,"; } print MPIBASEFD "ierror)\n"; &PrintArgDecls( $routine, 0, "" ); @@ -425,7 +427,7 @@ foreach $routine (keys(%mpi_routines)) { # preferable to DOUBLE PRECISION (in cases where the user is permitted # to change the size of these basic types(!)). This script must produce # a standard-conforming file. The top-level configure (in mpich/configure) -# will replace DOUBLE PRECISION with REAL*8 if the Fortran compiler +# will replace DOUBLE PRECISION with REAL*8 if the Fortran compiler # supports REAL*8. if ($is_MPI) { print MPIBASEFD " @@ -530,9 +532,9 @@ if ($is_MPI) { "; } # Here's where we need to place the interface definitions for the functions -# that take vector or scalar arguments (startall, testall/any/some, +# that take vector or scalar arguments (startall, testall/any/some, # waitall/any/some, group_translate_ranks, etc.) -# For each such routine, we need to generate two entries. Here's the +# For each such routine, we need to generate two entries. Here's the # example for STARTALL: # subroutine MPI_STARTALL_S(c,r,ierr) # integer c,r,ierr @@ -550,51 +552,54 @@ print MPIBASEFD " END INTERFACE\n"; if ($buildScalarVector) { # Create the interface modules foreach my $routine (keys(%scalarVectorRoutines)) { - $ucname = uc($routine); - print MPIBASEFD " INTERFACE ${out_prefix}$ucname\n"; + $ucname = uc($routine); + print MPIBASEFD " INTERFACE ${out_prefix}$ucname\n"; print MPIBASEFD " MODULE PROCEDURE ${out_prefix}${ucname}_S\n"; print MPIBASEFD " MODULE PROCEDURE ${out_prefix}${ucname}_V\n"; - print MPIBASEFD " END INTERFACE ! ${out_prefix}$ucname\n\n"; - + print MPIBASEFD " END INTERFACE ! ${out_prefix}$ucname\n\n"; + } print MPIBASEFD "\n CONTAINS\n"; # This is much like the base name (interface) block code foreach my $routine (keys(%scalarVectorRoutines)) { - $ucname = uc($routine); - $args = $mpi_routines{$routine}; - @parms = split(/,/, $args ); - $svArgs = $scalarVectorRoutines{$routine}; - # The scalar version + $ucname = uc($routine); + $args = $mpi_routines{$routine}; + @parms = split(/,/, $args ); + $svArgs = $scalarVectorRoutines{$routine}; + + # The scalar version print MPIBASEFD " SUBROUTINE ${out_prefix}${ucname}_S("; for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; + print MPIBASEFD "v$i,"; } print MPIBASEFD "ierror)\n"; - &PrintArgDecls( $routine, 1, $svArgs ); - print MPIBASEFD " EXTERNAL ${out_prefix}${ucname}\n"; - print MPIBASEFD " call ${out_prefix}$ucname("; + &PrintArgDecls( $routine, 1, $svArgs ); + + print MPIBASEFD " EXTERNAL ${out_prefix}${ucname}\n"; + print MPIBASEFD " call ${out_prefix}$ucname("; for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; + print MPIBASEFD "v$i,"; } - print MPIBASEFD "ierror)\n"; + print MPIBASEFD "ierror)\n"; - print MPIBASEFD " END SUBROUTINE ${out_prefix}${ucname}_S\n\n"; + print MPIBASEFD " END SUBROUTINE ${out_prefix}${ucname}_S\n\n"; - # The vector version + # The vector version print MPIBASEFD " SUBROUTINE ${out_prefix}${ucname}_V("; for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; + print MPIBASEFD "v$i,"; } print MPIBASEFD "ierror)\n"; - &PrintArgDecls( $routine, 0, "" ); - print MPIBASEFD " EXTERNAL ${out_prefix}${ucname}\n"; - print MPIBASEFD " call ${out_prefix}$ucname("; + &PrintArgDecls( $routine, 0, "" ); + + print MPIBASEFD " EXTERNAL ${out_prefix}${ucname}\n"; + print MPIBASEFD " call ${out_prefix}$ucname("; for ($i=0; $i<=$#parms; $i++) { - print MPIBASEFD "v$i,"; + print MPIBASEFD "v$i,"; } - print MPIBASEFD "ierror)\n"; + print MPIBASEFD "ierror)\n"; - print MPIBASEFD " END SUBROUTINE ${out_prefix}${ucname}_V\n\n"; + print MPIBASEFD " END SUBROUTINE ${out_prefix}${ucname}_V\n\n"; } } @@ -641,11 +646,11 @@ $PRIVATEVAR = ""; #$BINDACCESS = ", BIND(C)"; #$BINDDEF =""; # Yet another problem. -# Because MPI_Count may be longer than a single (Fortran) INTEGER, +# Because MPI_Count may be longer than a single (Fortran) INTEGER, # alignment restrictions may introduce padding in the structure # And one more problem: If a Fortran INTEGER is not the same as a C int, -# then these are also wrong (see the "fint" option in -# src/binding/f77/buildiface +# then these are also wrong (see the "fint" option in +# src/binding/f77/buildiface print MPIFD <<EOT; TYPE$BINDACCESS :: MPI_Status $BINDDEF @@ -655,11 +660,11 @@ print MPIFD <<EOT; END TYPE MPI_Status EOT -%handles = ( 'comm' => 'Comm', 'datatype' => 'Datatype', - 'group' => 'Group', 'win' => 'Win', - 'file' => 'File', 'op' => 'Op', 'errhandler' => 'Errhandler', - 'request' => 'Request', 'message' => 'Message', - 'info' => 'Info' ); +%handles = ( 'comm' => 'Comm', 'datatype' => 'Datatype', + 'group' => 'Group', 'win' => 'Win', + 'file' => 'File', 'op' => 'Op', 'errhandler' => 'Errhandler', + 'request' => 'Request', 'message' => 'Message', + 'info' => 'Info' ); foreach $handle (keys(%handles)) { $mpitype = $handles{$handle}; @@ -715,10 +720,10 @@ close MPIFD; # # Generate the choice argument routines -# FIXME: This file is not quite right. Also note that it is -# *input* for yet another step, one that generates particular values -# for the types of the choice arguments. We should consider using -# a different extension for this file, such as sed or in, so that +# FIXME: This file is not quite right. Also note that it is +# *input* for yet another step, one that generates particular values +# for the types of the choice arguments. We should consider using +# a different extension for this file, such as sed or in, so that # it is clearly not a ready-to-use Fortran 90 input file. # In particular, it needs to be set up so that # <typesize> @@ -765,18 +770,18 @@ foreach $routine (keys(%mpi_choice_routines)) { print MPIFD " SUBROUTINE ${out_prefix}${ucname}_T("; for ($i=0; $i<=$#parms; $i++) { - print MPIFD "v$i,"; + print MPIFD "v$i,"; } print MPIFD "ierror)\n"; if (defined($NeedConstants{$routine})) { - print MPIFD " USE ${out_prefix}CONSTANTS,ONLY:"; - $sep = ""; - foreach $name (split(/\s+/,$NeedConstants{$routine})) { - print MPIFD "$sep$name"; - $sep = ", "; - } - print MPIFD "\n"; + print MPIFD " USE ${out_prefix}CONSTANTS,ONLY:"; + $sep = ""; + foreach $name (split(/\s+/,$NeedConstants{$routine})) { + print MPIFD "$sep$name"; + $sep = ", "; + } + print MPIFD "\n"; } # print the arg decls ... @@ -784,49 +789,50 @@ foreach $routine (keys(%mpi_choice_routines)) { # including scalar. $nchoice = 0; for ($i=0; $i<=$#parms; $i++) { - $parms[$i] =~ s/^const\s//; # Remove const if present - $parm = $parms[$i]; - # Check for special args - $loc = $i+1; - if (defined($special_args{"$routine-$loc"})) { - $parm = $special_args{"$routine-$loc"}; - } - - if ($parm =~ /void/) { - # An alternative to this is to have a separate file for - # routines with 2 choice arguments - if ($nchoice == 0) { - print MPIFD " <type> v$i<dims>\n"; - } - else { - print MPIFD " <type$nchoice> v$i<dims$nchoice>\n"; - } - $nchoice ++; - } - else { - # Map the C type to the Fortran type - $cparm = $parm; - $cparm =~ s/\s+//g; - $fparm = $parmc2f{$cparm}; - if ($fparm eq "") { - print STDERR "$routine: No parm type for $cparm ($parm)\n"; - } - if ($fparm =~ /%name%/) { - $fparm =~ s/%name%/v$i/; - # In the name case, convert any %nl% to newlines and spaces - $fparm =~ s/%nl%/\n /g; - print MPIFD " $fparm\n"; - } - else { - print MPIFD " $fparm v$i\n"; - } - } + $parms[$i] =~ s/^const\s//; # Remove const if present + $parm = $parms[$i]; + # Check for special args + $loc = $i+1; + if (defined($special_args{"$routine-$loc"})) { + $parm = $special_args{"$routine-$loc"}; + } + + if ($parm =~ /void/) { + # An alternative to this is to have a separate file for + # routines with 2 choice arguments + if ($nchoice == 0) { + print MPIFD " <type> v$i<dims>\n"; + } + else { + print MPIFD " <type$nchoice> v$i<dims$nchoice>\n"; + } + $nchoice ++; + } + else { + # Map the C type to the Fortran type + $cparm = $parm; + $cparm =~ s/\s+//g; + $fparm = $parmc2f{$cparm}; + if ($fparm eq "") { + print STDERR "$routine: No parm type for $cparm ($parm)\n"; + } + if ($fparm =~ /%name%/) { + $fparm =~ s/%name%/v$i/; + # In the name case, convert any %nl% to newlines and spaces + $fparm =~ s/%nl%/\n /g; + print MPIFD " $fparm\n"; + } + else { + print MPIFD " $fparm v$i\n"; + } + } } + print MPIFD " INTEGER ierror\n"; print MPIFD " EXTERNAL ${out_prefix}${ucname}\n"; print MPIFD " CALL ${out_prefix}${ucname}("; for ($i=0; $i<=$#parms; $i++) { - print MPIFD "v$i,"; + print MPIFD "v$i,"; } print MPIFD "ierror)\n"; print MPIFD " END SUBROUTINE ${out_prefix}${ucname}_T\n\n"; @@ -878,7 +884,7 @@ if MAINTAINER_MODE \$(top_srcdir)/src/binding/f90/Makefile.mk: src/binding/f90/buildiface-stamp src/binding/f90/buildiface-stamp: \$(top_srcdir)/src/binding/f90/buildiface \$(top_srcdir)/src/include/mpi.h.in - ( cd \$(top_srcdir)/src/binding/f90 && ./buildiface ) +\t( cd \$(top_srcdir)/src/binding/f90 && ./buildiface ) endif MAINTAINER_MODE # variables for custom "silent-rules" for F90 modules @@ -953,7 +959,7 @@ EOT print MAKEFD <<EOT; # FIXME: We may want to edit the mpif.h to convert Fortran77-specific -# items (such as an integer*8 used for file offsets) into the +# items (such as an integer*8 used for file offsets) into the # corresponding Fortran 90 KIND type, to accomodate compilers that # reject non-standard features such as integer*8 (such as the Intel # Fortran compiler with -std95). @@ -1014,16 +1020,16 @@ mpi_f90_modules = \\ modinc_HEADERS = \$(mpi_f90_modules) # We need a free-format version of mpif.h with no external commands, -# including no wtime/wtick (removing MPI_WTICK also removes MPI_WTIME, +# including no wtime/wtick (removing MPI_WTICK also removes MPI_WTIME, # but leave MPI_WTIME_IS_GLOBAL). -# Also allow REAL*8 or DOUBLE PRECISION for the MPI_WTIME/MPI_WTICK +# Also allow REAL*8 or DOUBLE PRECISION for the MPI_WTIME/MPI_WTICK # declarations src/binding/f90/mpifnoext.h: src/binding/f77/mpif.h - rm -f \$@ - sed -e 's/^C/\\!/g' -e '/EXTERNAL/d' \\ - -e '/REAL\\*8/d' \\ - -e '/DOUBLE PRECISION/d' \\ - -e '/MPI_WTICK/d' src/binding/f77/mpif.h > \$@ +\trm -f \$@ +\tsed -e 's/^C/\\!/g' -e '/EXTERNAL/d' \\ +\t-e '/REAL\\*8/d' \\ +\t-e '/DOUBLE PRECISION/d' \\ +\t-e '/MPI_WTICK/d' src/binding/f77/mpif.h > \$@ CLEANFILES += src/binding/f90/mpifnoext.h @@ -1050,9 +1056,9 @@ print MAKEFD "\n"; # ---------------------------------------------------------------------------- # FIXME: Add the steps to handle the choice arguments. They should be -# selected by configure from a list of possible choices, with an -# enable switch used to bypass the checks. In addition, we need a way to -# dynamically create subsets, given a list of routines and types/dimensions +# selected by configure from a list of possible choices, with an +# enable switch used to bypass the checks. In addition, we need a way to +# dynamically create subsets, given a list of routines and types/dimensions # to include. This allows users to build precisely tailored Fortran90 modules. # ---------------------------------------------------------------------------- @@ -1080,12 +1086,12 @@ sub print_line { my $count = $_[2]; my $continue = $_[3]; my $continue_len = $_[4]; - + $linelen = length( $line ); #print "linelen = $linelen, print_line_len = $print_line_len\n"; if ($print_line_len + $linelen > $count) { - print $FD $continue; - $print_line_len = $continue_len; + print $FD $continue; + $print_line_len = $continue_len; } print $FD $line; $print_line_len += $linelen; @@ -1104,7 +1110,7 @@ sub print_endline { # config.status. sub createModSteps { my ($module, $deps, $srcFile, $use_srcdir) = @_; - + # Get a version of the source file with $(FCEXT) instead of .f90 # as the extension my $srcFileWithExt = $srcFile; @@ -1131,7 +1137,7 @@ sub createModSteps { # Attempt to deal with Fortran module files in a mostly sane way. Quick # overview for the less Fortran literate: - # + # # MPICH has four Fortran modules: mpi, mpi_constants, mpi_sizeofs, and # mpi_base. Each module is produced as a side effect of compiling the # corresponding .f90 file into a .lo. The .lo is produced by libtool and is @@ -1214,7 +1220,7 @@ EOT } -# Print the declarations for the given routine. +# Print the declarations for the given routine. sub PrintArgDecls { my ($routine,$svflag,$svArgs) = @_; @@ -1222,111 +1228,111 @@ sub PrintArgDecls { my $args = $mpi_routines{$routine}; my @parms = split(/,/, $args ); - # preload the svargs if requested. This is used to decide whether + # preload the svargs if requested. This is used to decide whether # an array arg is output as a scalar or a vector my %svargs = (); if ($svflag) { - for my $val (split(/:/,$svArgs)) { - my $loc = $val; - my $count = "-1"; - if ($loc =~ /(\d+)-(\d+)/) { - $loc = $1; - $count = $2; - } - $svargs{$loc} = $count; - } + for my $val (split(/:/,$svArgs)) { + my $loc = $val; + my $count = "-1"; + if ($loc =~ /(\d+)-(\d+)/) { + $loc = $1; + $count = $2; + } + $svargs{$loc} = $count; } - # Determine if we need any constants (e.g., MPI_STATUS_SIZE, + } + # Determine if we need any constants (e.g., MPI_STATUS_SIZE, # MPI_OFFSET_KIND) my %use_constants = (); my $found_constants = 0; for (my $i=0; $i<=$#parms; $i++) { - $parms[$i] =~ s/^const\s+//; # Remove const if present - $parm = $parms[$i]; - # Check for special args - $loc = $i+1; - if (defined($special_args{"$routine-$loc"})) { - $parm = $special_args{"$routine-$loc"}; - } - # Map the C type to the Fortran type - $cparm = $parm; - $cparm =~ s/\s+//g; - $fparm = $parmc2f{$cparm}; - # Now, does this type contain an MPI constant? - if (!defined($fparm)) { - print "$cparm value has no matching fortran parm\n"; - } - if ($fparm =~ /(MPI_[A-Z_]*)/) { - $use_constants{$1} = 1; - $found_constants = 1; - } + $parms[$i] =~ s/^const\s+//; # Remove const if present + $parm = $parms[$i]; + # Check for special args + $loc = $i+1; + if (defined($special_args{"$routine-$loc"})) { + $parm = $special_args{"$routine-$loc"}; + } + # Map the C type to the Fortran type + $cparm = $parm; + $cparm =~ s/\s+//g; + $fparm = $parmc2f{$cparm}; + # Now, does this type contain an MPI constant? + if (!defined($fparm)) { + print "$cparm value has no matching fortran parm\n"; + } + if ($fparm =~ /(MPI_[A-Z_]*)/) { + $use_constants{$1} = 1; + $found_constants = 1; + } } if ($found_constants) { - print MPIBASEFD " USE MPI_CONSTANTS,ONLY:"; - $sep = ""; - foreach $name (keys(%use_constants)) { - print MPIBASEFD "$sep$name"; - $sep = ", "; - $NeedConstants{$routine} .= "$name "; - } - print MPIBASEFD "\n"; + print MPIBASEFD " USE MPI_CONSTANTS,ONLY:"; + $sep = ""; + foreach $name (keys(%use_constants)) { + print MPIBASEFD "$sep$name"; + $sep = ", "; + $NeedConstants{$routine} .= "$name "; + } + print MPIBASEFD "\n"; } # Output argument types $lastParmType = "<none>"; for (my $i=0; $i<=$#parms; $i++) { - $parm = $parms[$i]; - # Check for special args - $loc = $i+1; - if (defined($special_args{"$routine-$loc"})) { - $parm = $special_args{"$routine-$loc"}; - } - # Map the C type to the Fortran type - $cparm = $parm; - $cparm =~ s/\s+//g; - $fparm = $parmc2f{$cparm}; - if ($fparm eq "") { - print STDERR "$routine: No parm type for $cparm ($parm)\n"; - } - # Split out the base type from the name - if ($fparm =~ /(\w+.*)\s+(%name\S.*)/) { - $parmType = $1; - $varName = $2; - if ($varName =~ /%name%/) { - $varName =~ s/%name%/v$i/; - } - $varName =~ s/%nl%/\n /g; - $parmType =~ s/%nl%/\n /g; - - # Here's where we might change vector to scalar args - if ($svflag) { - if (defined($svargs{$loc})) { - # The value is the count arg for the array; later, we - # can make use of that to improve the definitions - if ($varName =~ /,\*/) { - $varName =~ s/,\*//; - } - elsif ($varName =~ /$\*$/) { - $varName =~ s/$\*$//; - } - else { - print STDERR "Failed to make arg $i in $routine a scalar\n"; - } - } - } - } - else { - $parmType = $fparm; - $varName = "v$i"; - } - if ($parmType ne $lastParmType) { - if ($lastParmType ne "<none>" ) { print MPIBASEFD "\n"; } - print MPIBASEFD " $parmType $varName"; - $lastParmType = $parmType; - } - else { - print MPIBASEFD ", $varName"; - } + $parm = $parms[$i]; + # Check for special args + $loc = $i+1; + if (defined($special_args{"$routine-$loc"})) { + $parm = $special_args{"$routine-$loc"}; + } + # Map the C type to the Fortran type + $cparm = $parm; + $cparm =~ s/\s+//g; + $fparm = $parmc2f{$cparm}; + if ($fparm eq "") { + print STDERR "$routine: No parm type for $cparm ($parm)\n"; + } + # Split out the base type from the name + if ($fparm =~ /(\w+.*)\s+(%name\S.*)/) { + $parmType = $1; + $varName = $2; + if ($varName =~ /%name%/) { + $varName =~ s/%name%/v$i/; + } + $varName =~ s/%nl%/\n /g; + $parmType =~ s/%nl%/\n /g; + + # Here's where we might change vector to scalar args + if ($svflag) { + if (defined($svargs{$loc})) { + # The value is the count arg for the array; later, we + # can make use of that to improve the definitions + if ($varName =~ /,\*/) { + $varName =~ s/,\*//; + } + elsif ($varName =~ /$\*$/) { + $varName =~ s/$\*$//; + } + else { + print STDERR "Failed to make arg $i in $routine a scalar\n"; + } + } + } + } + else { + $parmType = $fparm; + $varName = "v$i"; + } + if ($parmType ne $lastParmType) { + if ($lastParmType ne "<none>" ) { print MPIBASEFD "\n"; } + print MPIBASEFD " $parmType $varName"; + $lastParmType = $parmType; + } + else { + print MPIBASEFD ", $varName"; + } } if ($lastParmType ne "<none>" ) { print MPIBASEFD "\n"; } @@ -1335,31 +1341,31 @@ sub PrintArgDecls { # # Replace old file with new file only if new file is different -# Otherwise, remove new filename +# Otherwise, remove new filename sub ReplaceIfDifferent { my ($oldfilename,$newfilename) = @_; my $rc = 1; - if (-s $oldfilename) { - $rc = system "cmp -s $newfilename $oldfilename"; - $rc >>= 8; # Shift right to get exit status + if (-s $oldfilename) { + $rc = system "cmp -s $newfilename $oldfilename"; + $rc >>= 8; # Shift right to get exit status } if ($rc != 0) { - # The files differ. Replace the old file - # with the new one - if (-s $oldfilename) { - print STDERR "Replacing $oldfilename\n"; - # If debugging and there is a difference, show that difference - if ($gDebug) { system "diff $newfilename $oldfilename"; } - - unlink $oldfilename; - } - else { - print STDERR "Creating $oldfilename\n"; - } - rename $newfilename, $oldfilename || - die "Could not replace $oldfilename"; + # The files differ. Replace the old file + # with the new one + if (-s $oldfilename) { + print STDERR "Replacing $oldfilename\n"; + # If debugging and there is a difference, show that difference + if ($gDebug) { system "diff $newfilename $oldfilename"; } + + unlink $oldfilename; + } + else { + print STDERR "Creating $oldfilename\n"; + } + rename $newfilename, $oldfilename || + die "Could not replace $oldfilename"; } else { - unlink $newfilename; + unlink $newfilename; } } http://git.mpich.org/mpich.git/commitdiff/80419d6d3b1a658d88886b645654e2297… commit 80419d6d3b1a658d88886b645654e2297fa28321 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 14:13:13 2013 -0600 Make ReadInterface extract both arg types & names Revised Perl subroutine ReadInterface(), which is used to extract function prototypes from a file. Previously, ReadInterface only extracted argument types of C functions. With this revise, it also extracts argument names. This is needed to support Fortran keyword arguments. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/binding/f90/binding.sub b/src/binding/f90/binding.sub index 7e534a7..ad4d00f 100755 --- a/src/binding/f90/binding.sub +++ b/src/binding/f90/binding.sub @@ -43,7 +43,9 @@ sub ReadInterface { print STDERR "Error in processing comment within interface file $prototype_file in line $origline"; } } - print "binding: read $_" if $gDebug; + + # Parse all routines returning an error code + print "\nParsing : $_" if $gDebug; if (/^int\s+$routine_prefix($routine_pattern)\s*\((.*)/) { $routine_name = $1; $args = $2; @@ -56,7 +58,7 @@ sub ReadInterface { # remove qualifiers from args $args =~ s/\s*const\s+//g; - print "binding: $routine_name ( $args )\n" if $gDebug; + print "Found : $routine_name($args)\n" if $gDebug; # Eventually, we'll create a new file here. # For C++, we may create similar files by looking up # the corresponding routines. @@ -65,27 +67,32 @@ sub ReadInterface { print STDERR "Duplicate prototypes for $routine_name\n"; next; } - # Clear variables - $args = &clean_args( $args ); + # Seperate argument types and names + my ($argtypes, $argnames) = &separate_args_and_append_ierror($args); # # Handle special cases # my $testname = $routine_name . "_args"; # if (defined($$testname)) { # print "replacing args for $routine_name\n" if $gDebug; # $args = $$testname; # } - $$routine_hash{$routine_name} = $args; + $$routine_hash{$routine_name} = [$argtypes, $argnames]; } } } # -# Look through $args for parameter names (foo\s+name) -# and remove them -sub clean_args { - my $args = $_[0]; - my $newargs = ""; - my $comma = ""; - for my $parm (split(',',$args)) { +# Look through $args, separate type and name of each argument. Then group argument +# types and names separately. In addition, append "int", "ierror" to existing +# argument types and names respectively. +# Input: an argument string +# Output: ($argtypes, $argnames), in which types (names) are delimited by comma. +sub separate_args_and_append_ierror { + my $argtypes = ""; + my $argnames = ""; + my $comma = ""; # no comma before the first arg + for my $parm (split(',', $_[0])) { + my $argtype = ""; + my $argname = ""; # Remove any leading or trailing spaces $parm =~ s/^\s*//; $parm =~ s/\s*$//; @@ -95,36 +102,51 @@ sub clean_args { $qualifier = "const "; $parm = $1; } + + if ($parm eq "void") { # e.g., MPI_Finalize(void) + last; + } + # Question: What to do with the qualifier? # Handle parameters with parameter names # 1. "int foo" # 2. "int *foo" # 3. "int foo[]" - # 4. "int *foo[]" - if ( ($parm =~ /^([A-Za-z0-9_]+)\s+[A-Za-z0-9_]+$/) ) { - $parm = $1; + # 4. "int *foo[]" or "int **foo[]" + if ( ($parm =~ /^([A-Za-z0-9_]+)\s+([A-Za-z0-9_]+)$/) ) { + $argtype = $1; + $argname = $2; } - elsif ( ($parm =~ /^([A-Za-z0-9_]+\s*\*)\s*[A-Za-z0-9_]+$/) ) { - $parm = $1; + elsif ( ($parm =~ /^([A-Za-z0-9_]+\s*\*)\s*([A-Za-z0-9_]+)$/) ) { + $argtype = $1; + $argname = $2; } - elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s*[A-Za-z0-9_]+(\[.*\])\s*$/) ) { - my $basename = $1; - my $arrayarg = $2; - #if ($arrayarg =~ /\[\s*\]/) { $arrayarg = "*"; } - $parm = $basename . $arrayarg; + elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s*([A-Za-z0-9_]+)(\[.*\])\s*$/) ) { + my $basetype = $1; + my $arraytype = $3; + #if ($arraytype =~ /\[\s*\]/) { $arraytype = "*"; } + $argtype = $basetype . $arraytype; + $argname = $2; } - elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s(\*?\*?)\s*[A-Za-z0-9_]+(\[.*\])\s*$/) ) { - my $basename = $1; - my $arrayarg = $2 . $3; - #if ($arrayarg =~ /\[\s*\]/) { $arrayarg = "*"; } - $parm = $basename . $arrayarg; + elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s(\*?\*?)\s*([A-Za-z0-9_]+)(\[.*\])\s*$/) ) { + my $basetype = $1; + my $arraytype = $2 . $4; + #if ($arraytype =~ /\[\s*\]/) { $arraytype = "*"; } + $argtype = $basetype . $arraytype; + $argname = $3; } - $newargs .= "$comma$parm"; + + $argtypes .= "$comma$argtype"; + $argnames .= "$comma$argname"; $comma = ","; } - print "$newargs\n" if $gDebug; - $args = $newargs; - return $args; + + # Append "int ierror" + $argtypes .= "${comma}int"; + $argnames .= "${comma}ierror"; + print "argtypes= $argtypes\n" if $gDebug; + print "argnames= $argnames\n" if $gDebug; + return ($argtypes, $argnames); } # Since this is a required package, indicate that we are successful. http://git.mpich.org/mpich.git/commitdiff/35a996524a0a58941829118f7eccac76a… commit 35a996524a0a58941829118f7eccac76a6ca3dba Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 13:53:40 2013 -0600 Add indentions, remove white spaces, replace Tabs Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/binding/f90/binding.sub b/src/binding/f90/binding.sub index 89feb62..7e534a7 100755 --- a/src/binding/f90/binding.sub +++ b/src/binding/f90/binding.sub @@ -1,12 +1,12 @@ #! /usr/bin/env perl # # This file contains common routines for reading a file of function prototypes -# (such as mpi.h) and extracting the function prototypes. +# (such as mpi.h) and extracting the function prototypes. -# +# # ReadInterface( filename, routineprefix, routinepattern, routinehash ) # Read file filename, look for routines that have a given prefix and name -# pattern, and insert that routine into routinehash with value the +# pattern, and insert that routine into routinehash with value the # arguments of the routine. #$Finalized_args = "bool"; @@ -17,64 +17,64 @@ sub ReadInterface { my $routine_pattern = $_[2]; my $routine_hash = $_[3]; # $debug is a global variable - + open( FD, "<$prototype_file" ) || die "Cannot open $prototype_file\n"; # Skip to prototypes while (<FD>) { - if ( /\/\*\s*Begin Prototypes/ ) { last; } + if ( /\/\*\s*Begin Prototypes/ ) { last; } } # Read each one while (<FD>) { - # Handle the special case of prototypes to ignore - if (/\/\*\s*Begin Skip Prototypes/) { - while (<FD>) { - if (/\/\*\s*End Skip Prototypes/) { last; } - } - } - if (/\/\*\s*End Prototypes/) { last; } - # Remove any comments - $origline = $_; - while (/(.*)\/\*(.*?)\*\/(.*)/) { - my $removed = $2; - $_ = $1.$3; - if ($2 =~ /\/\*/) { - print STDERR "Error in processing comment within interface file $prototype_file in line $origline"; - } - } - print "binding: read $_" if $gDebug; - if (/^int\s+$routine_prefix($routine_pattern)\s*$(.*)/) { - $routine_name = $1; - $args = $2; - while (! ($args =~ /;/)) { - $args .= <FD>; - } - $args =~ s/MPICH_ATTR[A-Z_]*\([^)]*$//g; - $args =~ s/\)\s*;//g; - $args =~ s/[\r\n]*//g; - # remove qualifiers from args - $args =~ s/\s*const\s+//g; - - print "binding: $routine_name ( $args )\n" if $gDebug; - # Eventually, we'll create a new file here. - # For C++, we may create similar files by looking up - # the corresponding routines. - # Check for duplicates in the list of routines - if (defined($$routine_hash{$routine_name})) { - print STDERR "Duplicate prototypes for $routine_name\n"; - next; - } - # Clear variables - $args = &clean_args( $args ); -# # Handle special cases -# my $testname = $routine_name . "_args"; -# if (defined($$testname)) { -# print "replacing args for $routine_name\n" if $gDebug; -# $args = $$testname; -# } - $$routine_hash{$routine_name} = $args; - } + # Handle the special case of prototypes to ignore + if (/\/\*\s*Begin Skip Prototypes/) { + while (<FD>) { + if (/\/\*\s*End Skip Prototypes/) { last; } + } + } + if (/\/\*\s*End Prototypes/) { last; } + # Remove any comments + $origline = $_; + while (/(.*)\/\*(.*?)\*\/(.*)/) { + my $removed = $2; + $_ = $1.$3; + if ($2 =~ /\/\*/) { + print STDERR "Error in processing comment within interface file $prototype_file in line $origline"; + } + } + print "binding: read $_" if $gDebug; + if (/^int\s+$routine_prefix($routine_pattern)\s*$(.*)/) { + $routine_name = $1; + $args = $2; + while (! ($args =~ /;/)) { + $args .= <FD>; + } + $args =~ s/MPICH_ATTR[A-Z_]*\([^)]*$//g; + $args =~ s/\)\s*;//g; + $args =~ s/[\r\n]*//g; + # remove qualifiers from args + $args =~ s/\s*const\s+//g; + + print "binding: $routine_name ( $args )\n" if $gDebug; + # Eventually, we'll create a new file here. + # For C++, we may create similar files by looking up + # the corresponding routines. + # Check for duplicates in the list of routines + if (defined($$routine_hash{$routine_name})) { + print STDERR "Duplicate prototypes for $routine_name\n"; + next; + } + # Clear variables + $args = &clean_args( $args ); + # # Handle special cases + # my $testname = $routine_name . "_args"; + # if (defined($$testname)) { + # print "replacing args for $routine_name\n" if $gDebug; + # $args = $$testname; + # } + $$routine_hash{$routine_name} = $args; + } } } @@ -86,41 +86,41 @@ sub clean_args { my $newargs = ""; my $comma = ""; for my $parm (split(',',$args)) { - # Remove any leading or trailing spaces - $parm =~ s/^\s*//; - $parm =~ s/\s*$//; - # Remove and remember qualifiers - $qualifier = ""; - if ($parm =~ /^const\s+(.*)/) { - $qualifier = "const "; - $parm = $1; - } - # Question: What to do with the qualifier? - # Handle parameters with parameter names - # 1. "int foo" + # Remove any leading or trailing spaces + $parm =~ s/^\s*//; + $parm =~ s/\s*$//; + # Remove and remember qualifiers + $qualifier = ""; + if ($parm =~ /^const\s+(.*)/) { + $qualifier = "const "; + $parm = $1; + } + # Question: What to do with the qualifier? + # Handle parameters with parameter names + # 1. "int foo" # 2. "int *foo" # 3. "int foo[]" # 4. "int *foo[]" - if ( ($parm =~ /^([A-Za-z0-9_]+)\s+[A-Za-z0-9_]+$/) ) { - $parm = $1; - } - elsif ( ($parm =~ /^([A-Za-z0-9_]+\s*\*)\s*[A-Za-z0-9_]+$/) ) { - $parm = $1; - } - elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s*[A-Za-z0-9_]+(\[.*\])\s*$/) ) { - my $basename = $1; - my $arrayarg = $2; - #if ($arrayarg =~ /\[\s*\]/) { $arrayarg = "*"; } - $parm = $basename . $arrayarg; - } - elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s(\*?\*?)\s*[A-Za-z0-9_]+(\[.*\])\s*$/) ) { - my $basename = $1; - my $arrayarg = $2 . $3; - #if ($arrayarg =~ /\[\s*\]/) { $arrayarg = "*"; } - $parm = $basename . $arrayarg; - } - $newargs .= "$comma$parm"; - $comma = ","; + if ( ($parm =~ /^([A-Za-z0-9_]+)\s+[A-Za-z0-9_]+$/) ) { + $parm = $1; + } + elsif ( ($parm =~ /^([A-Za-z0-9_]+\s*\*)\s*[A-Za-z0-9_]+$/) ) { + $parm = $1; + } + elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s*[A-Za-z0-9_]+(\[.*\])\s*$/) ) { + my $basename = $1; + my $arrayarg = $2; + #if ($arrayarg =~ /\[\s*\]/) { $arrayarg = "*"; } + $parm = $basename . $arrayarg; + } + elsif ( ($parm =~ /^([A-Za-z0-9_]+)\s(\*?\*?)\s*[A-Za-z0-9_]+(\[.*\])\s*$/) ) { + my $basename = $1; + my $arrayarg = $2 . $3; + #if ($arrayarg =~ /\[\s*\]/) { $arrayarg = "*"; } + $parm = $basename . $arrayarg; + } + $newargs .= "$comma$parm"; + $comma = ","; } print "$newargs\n" if $gDebug; $args = $newargs; ----------------------------------------------------------------------- Summary of changes: src/binding/f90/binding.sub | 214 +++++---- src/binding/f90/buildiface | 1161 ++++++++++++++++++++++-------------------- 2 files changed, 726 insertions(+), 649 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-98-g61f2600
by noreply＠mpich.org 19 Dec '13

19 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 61f260051e6244cf3986e8ee1b90a6cc67672b60 (commit) via 129adbf365319bc8921c249fb4abf5fc52b2761f (commit) via 8c133e469cbe28e90c07fa36cec89f9708d0ae2f (commit) via 6f0cdcc8b744cbe955b3bcb9964cbfebddebbaf9 (commit) via f43e6fe33beaf1eb94295621feabbebaf7d2566e (commit) via d96b1df370e308cbbceaa524cae7232e2c582226 (commit) via 6239007c3a9c415f5ebfc566788f3ca5eb6dd34a (commit) via c10aaa185e46c42968dbbaed77925f75e820cc80 (commit) from b2b9e1f9a5bbaf79126867531a7c685835319a04 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/61f260051e6244cf3986e8ee1b90a6cc6… commit 61f260051e6244cf3986e8ee1b90a6cc67672b60 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Sat Nov 16 06:44:41 2013 -0600 Add the missing NEMESIS_LMT_DMA_THRESHOLD Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c index a77e801..e2f8512 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c @@ -6,6 +6,24 @@ #include "mpid_nem_impl.h" #include "mpid_nem_datatypes.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_NEMESIS_LMT_DMA_THRESHOLD + category : NEMESIS + type : int + default : 2097152 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Messages larger than this size will use the "dma" (knem) + intranode LMT implementation, if it is enabled and available. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + MPIU_SUPPRESS_OSX_HAS_NO_SYMBOLS_WARNING; #if defined(HAVE_KNEM_IO_H) http://git.mpich.org/mpich.git/commitdiff/129adbf365319bc8921c249fb4abf5fc5… commit 129adbf365319bc8921c249fb4abf5fc52b2761f Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Nov 13 23:32:26 2013 -0600 Revise description of MPIR_CVAR_BCAST_* cvars Fixes #1928 Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpi/coll/bcast.c b/src/mpi/coll/bcast.c index 6edd389..bda2984 100644 --- a/src/mpi/coll/bcast.c +++ b/src/mpi/coll/bcast.c @@ -20,8 +20,16 @@ cvars: verbosity : MPI_T_VERBOSITY_USER_BASIC scope : MPI_T_SCOPE_ALL_EQ description : >- - the minimum number of processes in a communicator to use a non-binomial - broadcast algorithm + Let's define short messages as messages with size < MPIR_CVAR_BCAST_SHORT_MSG_SIZE, + and medium messages as messages with size >= MPIR_CVAR_BCAST_SHORT_MSG_SIZE but + < MPIR_CVAR_BCAST_LONG_MSG_SIZE, and long messages as messages with size >= + MPIR_CVAR_BCAST_LONG_MSG_SIZE. The broadcast algorithms selection procedure is + as follows. For short messages or when the number of processes is < + MPIR_CVAR_BCAST_MIN_PROCS, we do broadcast using the binomial tree algorithm. + Otherwise, for medium messages and with a power-of-two number of processes, we do + broadcast based on a scatter followed by a recursive doubling allgather algorithm. + Otherwise, for long messages or with non power-of-two number of processes, we do + broadcast based on a scatter followed by a ring allgather algorithm. - name : MPIR_CVAR_BCAST_SHORT_MSG_SIZE category : COLLECTIVE @@ -31,8 +39,17 @@ cvars: verbosity : MPI_T_VERBOSITY_USER_BASIC scope : MPI_T_SCOPE_ALL_EQ description : >- - the short message algorithm will be used if the send buffer size is < - this value (in bytes) + (Same description as that for MPIR_CVAR_BCAST_MIN_PROCS) + Let's define short messages as messages with size < MPIR_CVAR_BCAST_SHORT_MSG_SIZE, + and medium messages as messages with size >= MPIR_CVAR_BCAST_SHORT_MSG_SIZE but + < MPIR_CVAR_BCAST_LONG_MSG_SIZE, and long messages as messages with size >= + MPIR_CVAR_BCAST_LONG_MSG_SIZE. The broadcast algorithms selection procedure is + as follows. For short messages or when the number of processes is < + MPIR_CVAR_BCAST_MIN_PROCS, we do broadcast using the binomial tree algorithm. + Otherwise, for medium messages and with a power-of-two number of processes, we do + broadcast based on a scatter followed by a recursive doubling allgather algorithm. + Otherwise, for long messages or with non power-of-two number of processes, we do + broadcast based on a scatter followed by a ring allgather algorithm. - name : MPIR_CVAR_BCAST_LONG_MSG_SIZE category : COLLECTIVE @@ -42,8 +59,17 @@ cvars: verbosity : MPI_T_VERBOSITY_USER_BASIC scope : MPI_T_SCOPE_ALL_EQ description : >- - the long message algorithm will be used if the send buffer size is >= - this value (in bytes) + (Same description as that for MPIR_CVAR_BCAST_MIN_PROCS) + Let's define short messages as messages with size < MPIR_CVAR_BCAST_SHORT_MSG_SIZE, + and medium messages as messages with size >= MPIR_CVAR_BCAST_SHORT_MSG_SIZE but + < MPIR_CVAR_BCAST_LONG_MSG_SIZE, and long messages as messages with size >= + MPIR_CVAR_BCAST_LONG_MSG_SIZE. The broadcast algorithms selection procedure is + as follows. For short messages or when the number of processes is < + MPIR_CVAR_BCAST_MIN_PROCS, we do broadcast using the binomial tree algorithm. + Otherwise, for medium messages and with a power-of-two number of processes, we do + broadcast based on a scatter followed by a recursive doubling allgather algorithm. + Otherwise, for long messages or with non power-of-two number of processes, we do + broadcast based on a scatter followed by a ring allgather algorithm. - name : MPIR_CVAR_ENABLE_SMP_BCAST category : COLLECTIVE http://git.mpich.org/mpich.git/commitdiff/8c133e469cbe28e90c07fa36cec89f970… commit 8c133e469cbe28e90c07fa36cec89f9708d0ae2f Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Mon Nov 11 19:21:44 2013 -0600 Spread out cvar info into source files Fixes #1963 Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/include/mpiutil.h b/src/include/mpiutil.h index d6e764d..97417e2 100644 --- a/src/include/mpiutil.h +++ b/src/include/mpiutil.h @@ -6,6 +6,26 @@ #if !defined(MPIUTIL_H_INCLUDED) #define MPIUTIL_H_INCLUDED +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_NEMESIS_POLLS_BEFORE_YIELD + category : NEMESIS + type : int + default : 1000 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + When MPICH is in a busy waiting loop, it will periodically + call a function to yield the processor. This cvar sets + the number of loops before the yield function is called. A + value of 0 disables yielding. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + #ifndef HAS_MPID_ABORT_DECL /* FIXME: 4th arg is undocumented and bogus */ struct MPID_Comm; diff --git a/src/mpi/coll/allgather.c b/src/mpi/coll/allgather.c index a7fc8e3..99fef86 100644 --- a/src/mpi/coll/allgather.c +++ b/src/mpi/coll/allgather.c @@ -7,6 +7,35 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_ALLGATHER_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 81920 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + For MPI_Allgather and MPI_Allgatherv, the short message algorithm will + be used if the send buffer size is < this value (in bytes). + + - name : MPIR_CVAR_ALLGATHER_LONG_MSG_SIZE + category : COLLECTIVE + type : int + default : 524288 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + For MPI_Allgather and MPI_Allgatherv, the long message algorithm will be + used if the send buffer size is >= this value (in bytes) + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Allgather */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Allgather = PMPI_Allgather diff --git a/src/mpi/coll/allgatherv.c b/src/mpi/coll/allgatherv.c index f2faa14..275c404 100644 --- a/src/mpi/coll/allgatherv.c +++ b/src/mpi/coll/allgatherv.c @@ -7,6 +7,24 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_ALLGATHERV_PIPELINE_MSG_SIZE + category : COLLECTIVE + type : int + default : 32768 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + The smallest message size that will be used for the pipelined, large-message, + ring algorithm in the MPI_Allgatherv implementation. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Allgatherv */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Allgatherv = PMPI_Allgatherv diff --git a/src/mpi/coll/allreduce.c b/src/mpi/coll/allreduce.c index e0b014b..c2ac3b5 100644 --- a/src/mpi/coll/allreduce.c +++ b/src/mpi/coll/allreduce.c @@ -7,6 +7,55 @@ #include "mpiimpl.h" #include "collutil.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_ALLREDUCE_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 2048 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the short message algorithm will be used if the send buffer size is <= + this value (in bytes) + + - name : MPIR_CVAR_ENABLE_SMP_COLLECTIVES + category : COLLECTIVE + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Enable SMP aware collective communication. + + - name : MPIR_CVAR_ENABLE_SMP_ALLREDUCE + category : COLLECTIVE + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Enable SMP aware allreduce. + + - name : MPIR_CVAR_MAX_SMP_ALLREDUCE_MSG_SIZE + category : COLLECTIVE + type : int + default : 0 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Maximum message size for which SMP-aware allreduce is used. A + value of '0' uses SMP-aware allreduce for all message sizes. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Allreduce */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Allreduce = PMPI_Allreduce diff --git a/src/mpi/coll/alltoall.c b/src/mpi/coll/alltoall.c index 96e1f9b..8ec541a 100644 --- a/src/mpi/coll/alltoall.c +++ b/src/mpi/coll/alltoall.c @@ -7,6 +7,52 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories : + - name : COLLECTIVE + description : A category for collective communication variables. + +cvars: + - name : MPIR_CVAR_ALLTOALL_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 256 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the short message algorithm will be used if the per-destination + message size (sendcount*size(sendtype)) is <= this value + + - name : MPIR_CVAR_ALLTOALL_MEDIUM_MSG_SIZE + category : COLLECTIVE + type : int + default : 32768 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the medium message algorithm will be used if the per-destination + message size (sendcount*size(sendtype)) is <= this value and + larger than ALLTOALL_SHORT_MSG_SIZE + + - name : MPIR_CVAR_ALLTOALL_THROTTLE + category : COLLECTIVE + type : int + default : 32 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + max no. of irecvs/isends posted at a time in some alltoall + algorithms. Setting it to 0 causes all irecvs/isends to be + posted at once + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Alltoall */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Alltoall = PMPI_Alltoall diff --git a/src/mpi/coll/barrier.c b/src/mpi/coll/barrier.c index efcc4fd..d81e201 100644 --- a/src/mpi/coll/barrier.c +++ b/src/mpi/coll/barrier.c @@ -7,6 +7,22 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_ENABLE_SMP_BARRIER + category : COLLECTIVE + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : Enable SMP aware barrier. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Barrier */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Barrier = PMPI_Barrier diff --git a/src/mpi/coll/bcast.c b/src/mpi/coll/bcast.c index 6b56ad7..6edd389 100644 --- a/src/mpi/coll/bcast.c +++ b/src/mpi/coll/bcast.c @@ -8,6 +8,66 @@ #include "mpiimpl.h" #include "collutil.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_BCAST_MIN_PROCS + category : COLLECTIVE + type : int + default : 8 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the minimum number of processes in a communicator to use a non-binomial + broadcast algorithm + + - name : MPIR_CVAR_BCAST_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 12288 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the short message algorithm will be used if the send buffer size is < + this value (in bytes) + + - name : MPIR_CVAR_BCAST_LONG_MSG_SIZE + category : COLLECTIVE + type : int + default : 524288 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the long message algorithm will be used if the send buffer size is >= + this value (in bytes) + + - name : MPIR_CVAR_ENABLE_SMP_BCAST + category : COLLECTIVE + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : Enable SMP aware broadcast. + + - name : MPIR_CVAR_MAX_SMP_BCAST_MSG_SIZE + category : COLLECTIVE + type : int + default : 0 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Maximum message size for which SMP-aware broadcast is used. A + value of '0' uses SMP-aware broadcast for all message sizes. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Bcast */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Bcast = PMPI_Bcast diff --git a/src/mpi/coll/gather.c b/src/mpi/coll/gather.c index f7f9bbc..465c844 100644 --- a/src/mpi/coll/gather.c +++ b/src/mpi/coll/gather.c @@ -7,6 +7,35 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_GATHER_VSMALL_MSG_SIZE + category : COLLECTIVE + type : int + default : 1024 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + use a temporary buffer for intracommunicator MPI_Gather if the send + buffer size is < this value (in bytes) + + - name : MPIR_CVAR_GATHER_INTER_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 2048 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + use the short message algorithm for intercommunicator MPI_Gather if the + send buffer size is < this value (in bytes) + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Gather */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Gather = PMPI_Gather diff --git a/src/mpi/coll/gatherv.c b/src/mpi/coll/gatherv.c index c99b492..8deb6e3 100644 --- a/src/mpi/coll/gatherv.c +++ b/src/mpi/coll/gatherv.c @@ -7,6 +7,26 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_GATHERV_INTER_SSEND_MIN_PROCS + category : COLLECTIVE + type : int + default : 32 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Use Ssend (synchronous send) for intercommunicator MPI_Gatherv if the + "group B" size is >= this value. Specifying "-1" always avoids using + Ssend. For backwards compatibility, specifying "0" uses the default + value. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Gatherv */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Gatherv = PMPI_Gatherv diff --git a/src/mpi/coll/helper_fns.c b/src/mpi/coll/helper_fns.c index 8fbaedf..ac5ebf0 100644 --- a/src/mpi/coll/helper_fns.c +++ b/src/mpi/coll/helper_fns.c @@ -8,6 +8,32 @@ #include "mpiimpl.h" #include "datatype.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : FAULT_TOLERANCE + description : cvars that control fault tolerance behavior + +cvars: + - name : MPIR_CVAR_ENABLE_COLL_FT_RET + category : FAULT_TOLERANCE + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + DEPRECATED! Will be removed in MPICH-3.2 + Collectives called on a communicator with a failed process + should not hang, however the result of the operation may be + invalid even though the function returns MPI_SUCCESS. This + option enables an experimental feature that will return an error + if the result of the collective is invalid. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + #define COPY_BUFFER_SZ 16384 /* These functions are used in the implementation of collective diff --git a/src/mpi/coll/red_scat.c b/src/mpi/coll/red_scat.c index b04ea60..46dc14e 100644 --- a/src/mpi/coll/red_scat.c +++ b/src/mpi/coll/red_scat.c @@ -8,6 +8,24 @@ #include "mpiimpl.h" #include "collutil.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE + category : COLLECTIVE + type : int + default : 524288 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the long message algorithm will be used if the operation is commutative + and the send buffer size is >= this value (in bytes) + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Reduce_scatter */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Reduce_scatter = PMPI_Reduce_scatter diff --git a/src/mpi/coll/reduce.c b/src/mpi/coll/reduce.c index 0e3d7b7..c85fa53 100644 --- a/src/mpi/coll/reduce.c +++ b/src/mpi/coll/reduce.c @@ -8,6 +8,46 @@ #include "mpiimpl.h" #include "collutil.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_REDUCE_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 2048 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + the short message algorithm will be used if the send buffer size is <= + this value (in bytes) + + - name : MPIR_CVAR_ENABLE_SMP_REDUCE + category : COLLECTIVE + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Enable SMP aware reduce. + + - name : MPIR_CVAR_MAX_SMP_REDUCE_MSG_SIZE + category : COLLECTIVE + type : int + default : 0 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Maximum message size for which SMP-aware reduce is used. A + value of '0' uses SMP-aware reduce for all message sizes. + + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Reduce */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Reduce = PMPI_Reduce diff --git a/src/mpi/coll/scatter.c b/src/mpi/coll/scatter.c index ffd78f8..705a456 100644 --- a/src/mpi/coll/scatter.c +++ b/src/mpi/coll/scatter.c @@ -7,6 +7,24 @@ #include "mpiimpl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_SCATTER_INTER_SHORT_MSG_SIZE + category : COLLECTIVE + type : int + default : 2048 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + use the short message algorithm for intercommunicator MPI_Scatter if the + send buffer size is < this value (in bytes) + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Scatter */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Scatter = PMPI_Scatter diff --git a/src/mpi/comm/comm_split.c b/src/mpi/comm/comm_split.c index 379e792..83658bb 100644 --- a/src/mpi/comm/comm_split.c +++ b/src/mpi/comm/comm_split.c @@ -8,6 +8,27 @@ #include "mpiimpl.h" #include "mpicomm.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : COMMUNICATOR + description : cvars that control communicator construction and operation + +cvars: + - name : MPIR_CVAR_COMM_SPLIT_USE_QSORT + category : COMMUNICATOR + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Use qsort(3) in the implementation of MPI_Comm_split instead of bubble sort. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Comm_split */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Comm_split = PMPI_Comm_split diff --git a/src/mpi/comm/commutil.c b/src/mpi/comm/commutil.c index b759e79..dd2198d 100644 --- a/src/mpi/comm/commutil.c +++ b/src/mpi/comm/commutil.c @@ -17,6 +17,27 @@ #define MPID_COMM_PREALLOC 8 #endif +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_CTXID_EAGER_SIZE + category : THREADS + type : int + default : 2 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + The MPIR_CVAR_CTXID_EAGER_SIZE environment variable allows you to + specify how many words in the context ID mask will be set aside + for the eager allocation protocol. If the application is running + out of context IDs, reducing this value may help. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + + /* Preallocated comm objects */ /* initialized in initthread.c */ MPID_Comm MPID_Comm_builtin[MPID_COMM_N_BUILTIN] = { {0} }; diff --git a/src/mpi/debugger/dbginit.c b/src/mpi/debugger/dbginit.c index e1cad65..6ae6a21 100644 --- a/src/mpi/debugger/dbginit.c +++ b/src/mpi/debugger/dbginit.c @@ -11,6 +11,33 @@ #include <unistd.h> #endif +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_PROCTABLE_SIZE + category : DEBUGGER + type : int + default : 64 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Size of the "MPIR" debugger interface proctable (process table). + + - name : MPIR_CVAR_PROCTABLE_PRINT + category : DEBUGGER + type : boolean + default : false + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If true, dump the proctable entries at MPIR_WaitForDebugger-time. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* There are two versions of the debugger startup: 1. The debugger starts mpiexec - then mpiexec provides the MPIR_proctable information diff --git a/src/mpi/errhan/errutil.c b/src/mpi/errhan/errutil.c index dc275c3..84555e3 100644 --- a/src/mpi/errhan/errutil.c +++ b/src/mpi/errhan/errutil.c @@ -33,6 +33,38 @@ #include <stdio.h> /* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : ERROR_HANDLING + description : cvars that control error handling behavior (stack traces, aborts, etc) + +cvars: + - name : MPIR_CVAR_PRINT_ERROR_STACK + category : ERROR_HANDLING + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_LOCAL + description : >- + If true, print an error stack trace at error handling time. + + - name : MPIR_CVAR_CHOP_ERROR_STACK + category : ERROR_HANDLING + type : int + default : 0 + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_LOCAL + description : >- + If >0, truncate error stack output lines this many characters + wide. If 0, do not truncate, and if <0 use a sensible default. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + +/* * Structure of this file * * This file contains several groups of routines user for error handling diff --git a/src/mpi/init/finalize.c b/src/mpi/init/finalize.c index 19f3d9f..e22d0ff 100644 --- a/src/mpi/init/finalize.c +++ b/src/mpi/init/finalize.c @@ -8,6 +8,28 @@ #include "mpiimpl.h" #include "mpi_init.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : DEVELOPER + description : useful for developers working on MPICH itself + +cvars: + - name : MPIR_CVAR_MEMDUMP + category : DEVELOPER + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_MPIDEV_DETAIL + scope : MPI_T_SCOPE_LOCAL + description : >- + If true, list any memory that was allocated by MPICH and that + remains allocated when MPI_Finalize completes. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Finalize */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Finalize = PMPI_Finalize diff --git a/src/mpi/init/init.c b/src/mpi/init/init.c index 2ffcfce..56a87d0 100644 --- a/src/mpi/init/init.c +++ b/src/mpi/init/init.c @@ -7,6 +7,47 @@ #include "mpiimpl.h" #include "mpi_init.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : THREADS + description : multi-threading cvars + +cvars: + - name : MPIR_CVAR_ASYNC_PROGRESS + category : THREADS + type : boolean + default : false + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If set to true, MPICH will initiate an additional thread to + make asynchronous progress on all communication operations + including point-to-point, collective, one-sided operations and + I/O. Setting this variable will automatically increase the + thread-safety level to MPI_THREAD_MULTIPLE. While this + improves the progress semantics, it might cause a small amount + of performance overhead for regular MPI operations. The user + is encouraged to leave one or more hardware threads vacant in + order to prevent contention between the application threads + and the progress thread(s). The impact of oversubscription is + highly system dependent but may be substantial in some cases, + hence this recommendation. + + - name : MPIR_CVAR_DEFAULT_THREAD_LEVEL + category : THREADS + type : string + default : "MPI_THREAD_SINGLE" + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Sets the default thread level to use when using MPI_INIT. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ /* -- Begin Profiling Symbol Block for routine MPI_Init */ #if defined(HAVE_PRAGMA_WEAK) diff --git a/src/mpi/init/initthread.c b/src/mpi/init/initthread.c index 7401af7..3ebe996 100644 --- a/src/mpi/init/initthread.c +++ b/src/mpi/init/initthread.c @@ -24,6 +24,43 @@ #include <unistd.h> #endif +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : DEBUGGER + description : cvars relevant to the "MPIR" debugger interface + +cvars: + - name : MPIR_CVAR_DEBUG_HOLD + category : DEBUGGER + type : boolean + default : false + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If true, causes processes to wait in MPI_Init and + MPI_Initthread for a debugger to be attached. Once the + debugger has attached, the variable 'hold' should be set to 0 + in order to allow the process to continue (e.g., in gdb, "set + hold=0"). + + - name : MPIR_CVAR_ERROR_CHECKING + category : ERROR_HANDLING + type : boolean + default : true + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_LOCAL + description : >- + If true, perform checks for errors, typically to verify valid inputs + to MPI routines. Only effective when MPICH is configured with + --enable-error-checking=runtime . + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* -- Begin Profiling Symbol Block for routine MPI_Init_thread */ #if defined(HAVE_PRAGMA_WEAK) #pragma weak MPI_Init_thread = PMPI_Init_thread diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c index ae992ed..26ca89c 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c @@ -21,6 +21,69 @@ #include <signal.h> #endif +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_CH3_INTERFACE_HOSTNAME + category : CH3 + alt-env : MPIR_CVAR_INTERFACE_HOSTNAME + type : string + default : NULL + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_LOCAL + description : >- + If non-NULL, this cvar specifies the IP address that + other processes should use when connecting to this process. + This cvar is mutually exclusive with the + MPIR_CVAR_CH3_NETWORK_IFACE cvar and it is an error to set them + both. + + - name : MPIR_CVAR_CH3_PORT_RANGE + category : CH3 + alt-env : MPIR_CVAR_PORTRANGE, MPIR_CVAR_PORT_RANGE + type : range + default : "0:0" + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + The MPIR_CVAR_CH3_PORT_RANGE environment variable allows you to + specify the range of TCP ports to be used by the process + manager and the MPICH library. The format of this variable is + <low>:<high>. To specify any available port, use 0:0. + + - name : MPIR_CVAR_NEMESIS_TCP_NETWORK_IFACE + category : NEMESIS + alt-env : MPIR_CVAR_NETWORK_IFACE + type : string + default : NULL + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If non-NULL, this cvar specifies which pseudo-ethernet + interface the tcp netmod should use (e.g., "eth1", "ib0"). + Note, this is a Linux-specific cvar. + This cvar is mutually exclusive with the + MPIR_CVAR_CH3_INTERFACE_HOSTNAME cvar and it is an error to set + them both. + + - name : MPIR_CVAR_NEMESIS_TCP_HOST_LOOKUP_RETRIES + category : NEMESIS + type : int + default : 10 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + This cvar controls the number of times to retry the + gethostbyname() function before giving up. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + #define DBG_IFNAME 0 #ifdef ENABLE_CHECKPOINTING diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c index 96a6df1..82df5f3 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c @@ -6,6 +6,23 @@ #include "mpid_nem_impl.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === +cvars: + - name : MPIR_CVAR_NEMESIS_ENABLE_CKPOINT + category : NEMESIS + type : boolean + default : false + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If true, enables checkpointing support and returns an error if + checkpointing library cannot be initialized. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + MPIU_SUPPRESS_OSX_HAS_NO_SYMBOLS_WARNING; #ifdef ENABLE_CHECKPOINTING diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c index 5aa1f06..095ab20 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c @@ -11,6 +11,44 @@ #include "mpidi_nem_statistics.h" #include "mpit.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : NEMESIS + description : cvars that control behavior of the ch3:nemesis channel + +cvars: + - name : MPIR_CVAR_NEMESIS_SHM_EAGER_MAX_SZ + category : NEMESIS + type : int + default : -1 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + This cvar controls the message size at which Nemesis + switches from eager to rendezvous mode for shared memory. + If this cvar is set to -1, then Nemesis will choose + an appropriate value. + + - name : MPIR_CVAR_NEMESIS_SHM_READY_EAGER_MAX_SZ + category : NEMESIS + type : int + default : -2 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + This cvar controls the message size at which Nemesis + switches from eager to rendezvous mode for ready-send + messages. If this cvar is set to -1, then ready messages + will always be sent eagerly. If this cvar is set to -2, + then Nemesis will choose an appropriate value. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* constants for configure time selection of local LMT implementations */ #define MPID_NEM_LOCAL_LMT_NONE 0 #define MPID_NEM_LOCAL_LMT_SHM_COPY 1 diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c index fb93e43..468e58f 100644 --- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c +++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c @@ -9,6 +9,24 @@ #include "strings.h" #endif +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_NEMESIS_NETMOD + category : NEMESIS + type : string + default : "" + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If non-empty, this cvar specifies which network module + should be used for communication. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* initialize to prevent the compiler from generating common symbols */ MPID_nem_netmod_funcs_t *MPID_nem_netmod_func = NULL; int MPID_nem_netmod_id = -1; diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c index fe65823..f3c9759 100644 --- a/src/mpid/ch3/src/ch3u_rma_sync.c +++ b/src/mpid/ch3/src/ch3u_rma_sync.c @@ -7,6 +7,75 @@ #include "mpidimpl.h" #include "mpidrma.h" +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : CH3 + description : cvars that control behavior of ch3 + +cvars: + - name : MPIR_CVAR_CH3_RMA_ACC_IMMED + category : CH3 + type : boolean + default : true + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Use the immediate accumulate optimization + + - name : MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD + category : CH3 + type : int + default : 4000 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Threshold at which the RMA implementation attempts to complete requests + while completing RMA operations and while using the lazy synchonization + approach. Change this value if programs fail because they run out of + requests or other internal resources + + - name : MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD + category : CH3 + type : int + default : 128 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Threshold for the number of new requests since the last attempt to + complete pending requests. Higher values can increase performance, + but may run the risk of exceeding the available number of requests + or other internal resources. + + - name : MPIR_CVAR_CH3_RMA_LOCK_IMMED + category : CH3 + type : boolean + default : false + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Issue a request for the passive target RMA lock immediately. Default + behavior is to defer the lock request until the call to MPI_Win_unlock. + + - name : MPIR_CVAR_CH3_RMA_MERGE_LOCK_OP_UNLOCK + category : CH3 + type : boolean + default : true + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Enable/disable an optimization that merges lock, op, and unlock + messages, for single-operation passive target epochs. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_lockqueue_alloc); MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_winfence_clearlock); diff --git a/src/mpid/ch3/src/mpid_vc.c b/src/mpid/ch3/src/mpid_vc.c index 5640966..e35ed5d 100644 --- a/src/mpid/ch3/src/mpid_vc.c +++ b/src/mpid/ch3/src/mpid_vc.c @@ -22,6 +22,49 @@ #endif #include <ctype.h> +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +cvars: + - name : MPIR_CVAR_CH3_NOLOCAL + category : CH3 + alt-env : MPIR_CVAR_CH3_NO_LOCAL + type : boolean + default : false + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If true, force all processes to operate as though all processes + are located on another node. For example, this disables shared + memory communication hierarchical collectives. + + - name : MPIR_CVAR_CH3_ODD_EVEN_CLIQUES + category : CH3 + alt-env : MPIR_CVAR_CH3_EVEN_ODD_CLIQUES + type : boolean + default : false + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If true, odd procs on a node are seen as local to each other, and even + procs on a node are seen as local to each other. Used for debugging on + a single machine. + + - name : MPIR_CVAR_CH3_EAGER_MAX_MSG_SIZE + category : CH3 + type : int + default : 131072 + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + This cvar controls the message size at which CH3 switches + from eager to rendezvous mode. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ /*S * MPIDI_VCRT - virtual connection reference table diff --git a/src/nameserv/file/file_nameserv.c b/src/nameserv/file/file_nameserv.c index 3483dbc..2dc94b7 100644 --- a/src/nameserv/file/file_nameserv.c +++ b/src/nameserv/file/file_nameserv.c @@ -19,6 +19,31 @@ #endif #include <errno.h> +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : PROCESS_MANAGER + description : cvars that control the client-side process manager code + +cvars: + - name : MPIR_CVAR_NAMESERV_FILE_PUBDIR + category : PROCESS_MANAGER + alt-env : MPIR_CVAR_NAMEPUB_DIR + type : string + default : NULL + class : none + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + Sets the directory to use for MPI service publishing in the + file nameserv implementation. Allows the user to override + where the publish and lookup information is placed for + connect/accept based applications. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + /* For writing the name/service pair */ /* style: allow:fprintf:1 sig:0 */ diff --git a/src/util/mem/handlemem.c b/src/util/mem/handlemem.c index f565487..500ae8b 100644 --- a/src/util/mem/handlemem.c +++ b/src/util/mem/handlemem.c @@ -4,6 +4,33 @@ * See COPYRIGHT in top-level directory. */ +/* +=== BEGIN_MPI_T_CVAR_INFO_BLOCK === + +categories: + - name : MEMORY + description : affects memory allocation and usage, including MPI object handles + +cvars: + - name : MPIR_CVAR_ABORT_ON_LEAKED_HANDLES + category : MEMORY + type : boolean + default : false + class : device + verbosity : MPI_T_VERBOSITY_USER_BASIC + scope : MPI_T_SCOPE_ALL_EQ + description : >- + If true, MPI will call MPI_Abort at MPI_Finalize if any MPI object + handles have been leaked. For example, if MPI_Comm_dup is called + without calling a corresponding MPI_Comm_free. For uninteresting + reasons, enabling this option may prevent all known object leaks from + being reported. MPICH must have been configure with + "--enable-g=handlealloc" or better in order for this functionality to + work. + +=== END_MPI_T_CVAR_INFO_BLOCK === +*/ + #include "mpiimpl.h" #include <stdio.h> http://git.mpich.org/mpich.git/commitdiff/6f0cdcc8b744cbe955b3bcb9964cbfebd… commit 6f0cdcc8b744cbe955b3bcb9964cbfebddebbaf9 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 16:59:20 2013 -0600 Revert commented out code, to make a cvar work MPIR_CVAR_PROCTABLE_PRINT related code was originally commented out. Revert the code so that MPIR_CVAR_PROCTABLE_PRINT does what it means. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpi/debugger/dbginit.c b/src/mpi/debugger/dbginit.c index 788ceea..e1cad65 100644 --- a/src/mpi/debugger/dbginit.c +++ b/src/mpi/debugger/dbginit.c @@ -206,7 +206,6 @@ void MPIR_WaitForDebugger( void ) } MPIR_proctable_size = size; -#if 0 /* Debugging hook */ if (MPIR_CVAR_PROCTABLE_PRINT) { for (i=0; i<maxsize; i++) { @@ -215,7 +214,6 @@ void MPIR_WaitForDebugger( void ) } fflush( stdout ); } -#endif MPIR_Add_finalize( MPIR_FreeProctable, MPIR_proctable, 0 ); } else { http://git.mpich.org/mpich.git/commitdiff/f43e6fe33beaf1eb94295621feabbebaf… commit f43e6fe33beaf1eb94295621feabbebaf7d2566e Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 16:55:25 2013 -0600 Fix a wrong cvar name Added prefix MPIR_CVAR_ to GATHERV_INTER_SSEND_MIN_PROCS Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpi/coll/gatherv.c b/src/mpi/coll/gatherv.c index 8801e03..c99b492 100644 --- a/src/mpi/coll/gatherv.c +++ b/src/mpi/coll/gatherv.c @@ -145,7 +145,7 @@ int MPIR_Gatherv ( if (min_procs == -1) min_procs = comm_size + 1; /* Disable ssend */ else if (min_procs == 0) /* backwards compatibility, use default value */ - MPIR_CVAR_GET_DEFAULT_INT(GATHERV_INTER_SSEND_MIN_PROCS,&min_procs); + MPIR_CVAR_GET_DEFAULT_INT(MPIR_CVAR_GATHERV_INTER_SSEND_MIN_PROCS,&min_procs); if (comm_size >= min_procs) { mpi_errno = MPIC_Ssend(sendbuf, sendcount, sendtype, root, http://git.mpich.org/mpich.git/commitdiff/d96b1df370e308cbbceaa524cae7232e2… commit d96b1df370e308cbbceaa524cae7232e2c582226 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 16:42:48 2013 -0600 Implement extractcvars, which extracts cvar info. The script extractcvars extracts cvar info from C source files. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/autogen.sh b/autogen.sh index 97b84cb..c2f52e1 100755 --- a/autogen.sh +++ b/autogen.sh @@ -144,7 +144,7 @@ echo "done" # Default choices do_bindings=yes do_geterrmsgs=yes -do_getparms=yes +do_getcvars=yes do_f77=yes do_f77tof90=yes do_build_configure=yes @@ -841,13 +841,13 @@ fi echo "done" # new parameter code -echo_n "Generating parameter handling code... " -if test -x maint/gencvars -a "$do_getparms" = "yes" ; then - if ./maint/gencvars ; then +echo_n "Extracting control variables (cvar) ... " +if test -x maint/extractcvars -a "$do_getcvars" = "yes" ; then + if ./maint/extractcvars --dirs="`cat maint/cvardirs`"; then echo "done" else echo "failed" - error "unable to generate parameter handling code" + error "unable to extract control variables" exit 1 fi else diff --git a/maint/Makefile.mk b/maint/Makefile.mk index d44777d..d274836 100644 --- a/maint/Makefile.mk +++ b/maint/Makefile.mk @@ -25,7 +25,7 @@ dist_noinst_SCRIPTS += \ maint/gcovmerge.in \ maint/getcoverage.in \ maint/gen_subcfg_m4 \ - maint/gencvars \ + maint/extractcvars \ maint/genstates.in \ maint/getcoverage.in \ maint/local_perl/lib/YAML/Tiny.pm \ @@ -42,6 +42,7 @@ dist_noinst_DATA += \ maint/Version.base.m4 \ maint/docnotes \ maint/errmsgdirs \ + maint/cvardirs \ maint/gccimpgen.cpp \ maint/impgen.vcproj \ maint/local_perl/README \ diff --git a/maint/cvardirs b/maint/cvardirs new file mode 100644 index 0000000..04a2a67 --- /dev/null +++ b/maint/cvardirs @@ -0,0 +1 @@ +src/mpi src/mpi_t src/nameserv src/util src/binding src/include src/mpid src/pmi src/armci src/mutex diff --git a/maint/extractcvars b/maint/extractcvars index 7da93bd..f721123 100755 --- a/maint/extractcvars +++ b/maint/extractcvars @@ -4,26 +4,18 @@ # See COPYRIGHT in top-level directory. # -# script TODO: -# - generate initialization function to read vals from env vars -# - deal with string escaping in generated C strings -# - sort/collate cvars by name/category +# Parse C or header files to search for MPI_T control variable +# and category info, then output the info in source code form. use strict; use warnings; - -# help perl find the YAML parsing module -use lib 'maint/local_perl/lib'; - -use YAML::Tiny qw(); use File::Basename qw(basename); use Data::Dumper; use Getopt::Long; -# I'm pretty sure this is a standard lib module across all perl5 -# installs, but we can work around this easily if that doesn't turn out -# to be true. [goodell@ 2010-04-26] -use Digest::MD5 qw(); +# Help perl find the YAML parsing module +use lib './maint/local_perl/lib'; +use YAML::Tiny qw(); # To format README file use Text::Wrap; @@ -33,99 +25,125 @@ $Text::Wrap::unexpand = 0; # disable hard tabs in output # set true to enable debug output my $debug = 0; +my @cvars = (); +my @categories=(); +my $yaml = YAML::Tiny->new(); +my @dirs = (); +my @cfiles = (); +my %skipfiles = (); # namespace prefix for function names my $fn_ns = "MPIR_T_cvar"; - # namespace prefix for variable and type names my $ns = "MPIR_CVAR"; -# an alternative namespace used for environment variables, unused if set -# to "" +# an alternative namespace used for environment variables, unused if set to "" my $alt_ns = "MPIR_PARAM"; - # deprecated prefix for backward compatibility my $dep_ns = "MPICH"; -# cvar description file -my $cvar_file = "src/util/cvar/cvars.yml"; - -# output source files +# Default :output source files my $header_file = "src/include/mpich_cvars.h"; my $c_file = "src/util/cvar/mpich_cvars.c"; my $readme_file = "README.envvar"; -GetOptions( - "help!" => \&print_usage_and_exit, - "debug!" => \$debug, - "namespace=s" => \$ns, - "alt-namespace=s" => \$alt_ns, - "param-file" => \$cvar_file, - "header=s" => \$header_file, - "c-file=s" => \$c_file, - "readme-file=s" => \$readme_file, -) or die "unable to parse options, stopped"; - -sub print_usage_and_exit { +sub Usage { print <<EOT; + Usage: $0 [OPTIONS] Supported options: - - --help - this output - --debug - enable some debugging output - --namespace=STR - use STR as variable/type prefix in generated code - --param-file=FILE - use FILE as input describing cvars + --help - this output (optional) + --debug - enable some debugging output (optional) + --namespace=STR - use STR as variable/type prefix in generated code (optional, default=$ns) + --alt-namespace=STR - use STR as alternaive variable/type prefix in generated code (optional, default=$alt_ns) + --header=STR - specify the header file name (optional, default=$header_file) + --c-file=STR - specify the C file name (optional, default=$c_file) + --readme-file=STR - specify the readme file name (optional, default=$readme_file) + --dirs=STR - recursively parse C or header files under the directories as specified + --skip=STR - skip parsing files as specified (optional) EOT exit 1; } -my $run_timestamp = localtime; -my $uc_ns = uc($ns); +# Step 1: Process options +my $dirline; +my $skipline = ""; +GetOptions( + "help!" => \&Usage, + "debug!" => \$debug, + "namespace=s" => \$ns, + "alt-namespace=s" => \$alt_ns, + "header=s" => \$header_file, + "c-file=s" => \$c_file, + "readme-file=s" => \$readme_file, + "dirs=s" => \$dirline, + "skips=s" => \$skipline +) or Usage(); -######################################################################## -# read the config file and turn it into a perl hash/array object +Usage unless $dirline; -# NOTE: if multiple configuration files are supported in the future, -# this is the place that should be modified to read them all in and -# merge them into a single consistent configuration object +print "dirline = $dirline\n" if $debug; +@dirs = split(/[:,;\s]+/, $dirline); -my $yaml = YAML::Tiny->new(); -my $cvars = ($yaml->read($cvar_file))->[0]; # [0] is for the first document -print Dumper($cvars)."\n" if $debug; -die "not a HASH, stopped" unless ref($cvars) eq "HASH"; +my @skips = split(/[:,;\s]+/, $skipline); +%skipfiles = map {$_ => 1} @skips; + +# Step 2: Search all cfiles and put them in @cfiles. +foreach my $dir (@dirs) { + ExpandDir($dir); +} + +# Step 3: Parse each cfile and put results in @cvars and @categories. +foreach my $cfile (@cfiles) { + ProcessFile($cfile); +} -######################################################################## -# validate the config file +# Step 4: Preprocess cvars: +# *) Make sure that all categories referenced by cvars actually exist +# *) Strip out the prefix of their name (normally, MPIR_CVAR) +die "missing 'cvars', stopped" unless (@cvars); +my %cat_hash = map {$_->{name} => 1} @categories; -# only simple checks for now, just make sure that all categories -# referenced by cvars actually exist -my %cat_hash = (map { ($_->{name} => 1) } @{$cvars->{categories}}); -foreach my $p (@{$cvars->{cvars}}) { +foreach my $p (@cvars) { unless (exists $cat_hash{$p->{category}}) { - warn "category '".$p->{category}."' referenced by '".$p->{name}."' was not found"; + warn "Category '".$p->{category}."' referenced by '".$p->{name}."' was not found"; + } + $p->{name} =~ s/${ns}_//; + if (exists $p->{'alt-env'}) { + my @alts = split(/[:,;\s]+/, $p->{'alt-env'}); + foreach my $elmt (@alts) { + $elmt =~ s/${ns}_//; + } + $p->{'alt-env'} = [@alts]; } } -######################################################################## -# setup output files -open(CVAR_HDR, '>', $header_file); -open(CVAR_C, '>', $c_file); -open(CVAR_README, '>', $readme_file); +# Step 5: Output cvars and categories +print "Categories include: \n".Dumper(@categories) if $debug; +print "Cvars include :\n".Dumper(@cvars)."\n" if $debug; -my $hdr_guard = header_to_incl_guard($header_file); -my $cvar_file_md5 = md5sum($cvar_file); +my $run_timestamp = localtime; +my $uc_ns = uc($ns); + +# Setup output files +open(OUTPUT_H, '>', $header_file); +open(OUTPUT_C, '>', $c_file); +open(OUTPUT_README, '>', $readme_file); -print CVAR_HDR <<EOT; +#=============================================================== +# Step 5.1: Dump the header file. +my $hdr_guard = Header2InclGuard($header_file); +print OUTPUT_H <<EOT; /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2011 by Argonne National Laboratory. * See COPYRIGHT in top-level directory. */ -/* automatically generated + +/* Automatically generated * by: $0 * on: $run_timestamp - * from: $cvar_file (md5sum $cvar_file_md5) * * DO NOT EDIT!!! */ @@ -135,9 +153,35 @@ print CVAR_HDR <<EOT; #include "mpitimpl.h" /* for MPIR_T_cvar_range_value_t */ +/* Initializes cvar values from the environment */ +int ${fn_ns}_init(void); +int ${fn_ns}_finalize(void); + +/* Extern declarations for each cvar + * (definitions in $c_file) */ + +EOT + +foreach my $p (@cvars) { + printf OUTPUT_H "/* declared in $p->{location} */\n"; + printf OUTPUT_H "extern %s ${uc_ns}_%s;\n", Type2Ctype($p->{type}), $p->{name}; +} + +print OUTPUT_H <<EOT; + +/* TODO: this should be defined elsewhere */ +#define ${ns}_assert MPIU_Assert + +/* Arbitrary, simplifies interaction with external interfaces like MPI_T_ */ +#define ${uc_ns}_MAX_STRLEN (4096) + +#endif /* $hdr_guard */ EOT +close(OUTPUT_H); -print CVAR_C <<EOT; +#=============================================================== +# Step 5.2: Dump the C file. +print OUTPUT_C <<EOT; /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ /* * (C) 2011 by Argonne National Laboratory. @@ -146,71 +190,38 @@ print CVAR_C <<EOT; /* automatically generated * by: $0 * at: $run_timestamp - * from: $cvar_file (md5sum $cvar_file_md5) * * DO NOT EDIT!!! */ #include "mpiimpl.h" +/* Actual storage for cvars */ EOT - -print CVAR_README <<EOT; -(C) 2011 by Argonne National Laboratory. - See COPYRIGHT in top-level directory. - -This file lists the various environment variables available to change the -behavior of the MPICH library. These are intended to be used by advanced -users. ---------------------------------------------------------------------------- - -EOT - -######################################################################## -# now the actual cvars -die "missing 'cvars', stopped" unless exists $cvars->{cvars}; - -print CVAR_HDR <<EOT; -/* initializes cvar values from the environment */ -int ${fn_ns}_init(void); -int ${fn_ns}_finalize(void); - -/* extern declarations for each cvar - * (definitions in $c_file) */ -EOT - -# XXX DJG TODO collate and separate by category -foreach my $p (@{$cvars->{cvars}}) { - printf CVAR_HDR "extern %s ${uc_ns}_%s;\n", - type2ctype($p->{type}), $p->{name}; -} - -print CVAR_C "/* actual storage for cvars */\n"; - -foreach my $p (@{$cvars->{cvars}}) { - my $default; - if ($p->{type} eq "string") { - # handle strings specially to avoid various const issues - $default = fmt_default($p->{name}, undef, "NULL", $p->{type}); - } - else { - $default = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); +# Output the definitions +foreach my $p (@cvars) { + my $default = FmtDefault($p->{name}, $p->{default}, $p->{type}); + + if ($p->{class} eq 'device') { + printf OUTPUT_C "#if defined MPID_%s\n", $p->{name}; + printf OUTPUT_C "%s ${uc_ns}_%s = MPID_%s;\n", Type2Ctype($p->{type}), $p->{name}, + $p->{name}; + printf OUTPUT_C "#else\n"; } - if ($p->{class} eq "device") { - printf CVAR_C "#if defined MPID_%s\n", $p->{name}; - printf CVAR_C "%s ${uc_ns}_%s = MPID_%s;\n", type2ctype($p->{type}), $p->{name}, - $p->{name}; - printf CVAR_C "#else\n"; + if ($p->{type} eq 'string') { + printf OUTPUT_C "%s ${uc_ns}_%s = (char*)%s;\n", Type2Ctype($p->{type}), $p->{name}, $default; + } else { + printf OUTPUT_C "%s ${uc_ns}_%s = %s;\n", Type2Ctype($p->{type}), $p->{name}, $default; } - printf CVAR_C "%s ${uc_ns}_%s = %s;\n", type2ctype($p->{type}), $p->{name}, $default; - if ($p->{class} eq "device") { - printf CVAR_C "#endif /* MPID_%s */\n\n", $p->{name}; + + if ($p->{class} eq 'device') { + printf OUTPUT_C "#endif /* MPID_%s */\n\n", $p->{name}; } } -# FIXME the mpi_errno bit is MPICH-specific -print CVAR_C <<EOT; +# Generate the init function. +print OUTPUT_C <<EOT; #undef FUNCNAME #define FUNCNAME ${fn_ns}_init @@ -231,62 +242,58 @@ int ${fn_ns}_init(void) EOT -foreach my $p (@{$cvars->{cvars}}) { - my $count = 1; +# Register categories +foreach my $cat (@categories) { + my $desc = $cat->{description}; + $desc =~ s/"/\\"/g; + printf OUTPUT_C qq( /* declared in $cat->{location} */\n); + printf OUTPUT_C qq( MPIR_T_cat_add_desc(%s\n%s);\n\n), + qq("$cat->{name}",), + qq( "$desc"); +} + +# Register and init cvars +foreach my $p (@cvars) { + my $count; my $mpi_dtype; + my $dftval; + + # Set count and default value of the car if ($p->{type} eq 'string') { $mpi_dtype = "MPI_CHAR"; - $count = "MPIR_CVAR_MAX_STRLEN"; - my $str_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( defaultval.str = (char *)%s;\n), $str_val; + $count = "${uc_ns}_MAX_STRLEN"; + $dftval = FmtDefault($p->{name}, $p->{default}, $p->{type}); + printf OUTPUT_C qq( defaultval.str = (char *)%s;\n), $dftval; } elsif ($p->{type} eq 'int' or $p->{type} eq 'boolean') { $mpi_dtype = "MPI_INT"; $count = 1; - my $int_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( defaultval.d = %s;\n), $int_val; - } - elsif ($p->{type} eq 'unsigned') { - $mpi_dtype = "MPI_UNSINGED"; - $count = 1; - my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( defaultval.u = %s;\n), $double_val; - } - elsif ($p->{type} eq 'unsigned long') { - $mpi_dtype = "MPI_DOUBLE"; - $count = 1; - my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( defaultval.ul = %s;\n), $double_val; - } - elsif ($p->{type} eq 'unsigned long long') { - $mpi_dtype = "MPI_DOUBLE"; - $count = 1; - my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( defaultval.ull = %s;\n), $double_val; + $dftval = FmtDefault($p->{name}, $p->{default}, $p->{type}); + printf OUTPUT_C qq( defaultval.d = %s;\n), $dftval; } elsif ($p->{type} eq 'double') { $mpi_dtype = "MPI_DOUBLE"; $count = 1; - my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( defaultval.d = %s;\n), $double_val; + $dftval = FmtDefault($p->{name}, $p->{default}, $p->{type}); + printf OUTPUT_C qq( defaultval.d = %s;\n), $dftval; } elsif ($p->{type} eq 'range') { $mpi_dtype = "MPI_INT"; $count = 2; - my $range_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - printf CVAR_C qq( {\n); - printf CVAR_C qq( MPIR_T_cvar_range_value_t tmp = %s;\n), $range_val; - printf CVAR_C qq( defaultval.range = tmp;\n); - printf CVAR_C qq( }\n); + $dftval = FmtDefault($p->{name}, $p->{default}, $p->{type}); + printf OUTPUT_C qq( {\n); + printf OUTPUT_C qq( MPIR_T_cvar_range_value_t tmp = %s;\n), $dftval; + printf OUTPUT_C qq( defaultval.range = tmp;\n); + printf OUTPUT_C qq( }\n); } else { die "unknown type $p->{type}, stopped"; } + # Register the cvar my $desc = $p->{description}; $desc =~ s/"/\\"/g; - - printf CVAR_C qq( MPIR_T_CVAR_REGISTER_STATIC(\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s);\n), + printf OUTPUT_C qq( MPIR_T_CVAR_REGISTER_STATIC(\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s);\n), qq( $mpi_dtype,), qq( ${uc_ns}_$p->{name}, /* name */), qq( &${uc_ns}_$p->{name}, /* address */), @@ -298,17 +305,17 @@ foreach my $p (@{$cvars->{cvars}}) { qq( "$desc"); if ($p->{type} eq 'string') { -print CVAR_C <<EOT; +print OUTPUT_C <<EOT; ${uc_ns}_GET_DEFAULT_STRING(${uc_ns}_$p->{name}, &tmp_str); EOT } - my $env_fn = type_to_env_fn($p->{type}); + # Get the env variable value. + my $env_fn = Type2EnvFn($p->{type}); my @env_names = (); my $var_name = "${uc_ns}_" . $p->{name}; - # process extra envs first so the primary always wins - push @env_names, @{$p->{'abs-alt-env'}} if $p->{'abs-alt-env'}; + # Process extra envs first so the primary always wins push @env_names, map { "${dep_ns}_$_" } @{$p->{'alt-env'}}; push @env_names, map { "${alt_ns}_$_" } @{$p->{'alt-env'}}; push @env_names, map { "${uc_ns}_$_" } @{$p->{'alt-env'}}; @@ -319,26 +326,26 @@ EOT foreach my $env_name (@env_names) { # assumes rc is defined if ($p->{type} eq 'range') { - print CVAR_C <<EOT; +print OUTPUT_C <<EOT; rc = MPL_env2${env_fn}("$env_name", &($var_name.low), &($var_name.high)); MPIU_ERR_CHKANDJUMP1((-1 == rc),mpi_errno,MPI_ERR_OTHER,"**envvarparse","**envvarparse %s","$env_name"); EOT } elsif ($p->{type} eq 'string') { - print CVAR_C <<EOT; +print OUTPUT_C <<EOT; rc = MPL_env2${env_fn}("$env_name", &tmp_str); MPIU_ERR_CHKANDJUMP1((-1 == rc),mpi_errno,MPI_ERR_OTHER,"**envvarparse","**envvarparse %s","$env_name"); EOT } else { - print CVAR_C <<EOT; +print OUTPUT_C <<EOT; rc = MPL_env2${env_fn}("$env_name", &($var_name)); MPIU_ERR_CHKANDJUMP1((-1 == rc),mpi_errno,MPI_ERR_OTHER,"**envvarparse","**envvarparse %s","$env_name"); EOT } } if ($p->{type} eq 'string') { - print CVAR_C <<EOT; +print OUTPUT_C <<EOT; if (tmp_str != NULL) { ${var_name} = MPIU_Strdup(tmp_str); ${ns}_assert(${var_name}); @@ -352,18 +359,10 @@ EOT } EOT } - print CVAR_C "\n"; + print OUTPUT_C "\n"; } -foreach my $cat (@{$cvars->{categories}}) { - my $desc = $cat->{description}; - $desc =~ s/"/\\"/g; - printf CVAR_C qq( MPIR_T_cat_add_desc(%s\n%s);\n\n), - qq("$cat->{name}",), - qq( "$desc"); -} - -print CVAR_C <<EOT; +print OUTPUT_C <<EOT; fn_exit: return mpi_errno; fn_fail: @@ -372,19 +371,19 @@ fn_fail: EOT -print CVAR_C <<EOT; +print OUTPUT_C <<EOT; int ${fn_ns}_finalize(void) { int mpi_errno = MPI_SUCCESS; EOT -foreach my $p (@{$cvars->{cvars}}) { +foreach my $p (@cvars) { my $var_name = "${uc_ns}_" . $p->{name}; - if ($p->{type} eq "string") { - # need to cleanup after whatever was strduped by the init routine -print CVAR_C <<EOT; + if ($p->{type} eq 'string') { + # Need to cleanup after whatever was strduped by the init routine +print OUTPUT_C <<EOT; if (${var_name} != NULL) { MPIU_Free(${var_name}); ${var_name} = NULL; @@ -394,8 +393,7 @@ EOT } } - -print CVAR_C <<EOT; +print OUTPUT_C <<EOT; fn_exit: return mpi_errno; fn_fail: @@ -403,8 +401,27 @@ fn_fail: } EOT +close(OUTPUT_C); + +#=============================================================== +#Step 5.3 Dump the readme file +print OUTPUT_README <<EOT; +(C) 2011 by Argonne National Laboratory. + See COPYRIGHT in top-level directory. + +Automatically generated + by: $0 + at: $run_timestamp +DO NOT EDIT!!! -foreach my $p (@{$cvars->{cvars}}) { +This file lists the various environment variables available to change the +behavior of the MPICH library. These are intended to be used by advanced +users. +--------------------------------------------------------------------------- + +EOT + +foreach my $p (@cvars) { my @env_names = (); my $first; my $alt; @@ -416,64 +433,32 @@ foreach my $p (@{$cvars->{cvars}}) { push @env_names, map { "${uc_ns}_$_" } @{$p->{'alt-env'}}; push @env_names, map { "${alt_ns}_$_" } @{$p->{'alt-env'}}; push @env_names, map { "${dep_ns}_$_" } @{$p->{'alt-env'}}; - push @env_names, @{$p->{'abs-alt-env'}} if $p->{'abs-alt-env'}; - print CVAR_README "${uc_ns}_$p->{name}\n"; + print OUTPUT_README "${uc_ns}_$p->{name}\n"; $first = 1; foreach $alt (@env_names) { if ($first) { - print CVAR_README " Aliases: $alt\n"; + print OUTPUT_README " Aliases: $alt\n"; } else { - print CVAR_README " $alt\n"; + print OUTPUT_README " $alt\n"; } $first = 0; } - print CVAR_README wrap(" Description: ", " ", $p->{description} . "\n"); - $default = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type}); - print CVAR_README " Default: $default\n"; - print CVAR_README "\n"; + print OUTPUT_README wrap(" Description: ", " ", $p->{description} . "\n"); + $default = FmtDefault($p->{name}, $p->{default}, $p->{type}); + print OUTPUT_README " Default: $default\n"; + print OUTPUT_README "\n"; } +close(OUTPUT_README); -######################################################################## -# clean up - -close(CVAR_C); - -print CVAR_HDR <<EOT; - -/* TODO: this should be defined elsewhere */ -#define ${ns}_assert MPIU_Assert - -/* arbitrary, simplifies interaction with external interfaces like MPI_T_ */ -#define ${uc_ns}_MAX_STRLEN (4096) - -/* helper macros for safely getting the default value of a cvar */ -EOT - -print CVAR_HDR <<EOT; -#endif /* $hdr_guard */ -EOT -close(CVAR_HDR); - -print CVAR_README <<EOT; ---------------------------------------------------------------------------- +#=============================================================== +# Helper subroutines used in this script -Automatically generated - by: $0 - at: $run_timestamp - from: $cvar_file (md5sum $cvar_file_md5) - -EOT -close(CVAR_README); - -######################################################################## -# helper subroutines - -# transform a cvar type to a C-language type -sub type2ctype { +# Transform a cvar type to a C-language type +sub Type2Ctype { my $type = shift; my %typemap = ( 'int' => 'int', @@ -486,25 +471,18 @@ sub type2ctype { return $typemap{$type}; } -# transform a default value into a C value -sub fmt_default { +# Transform a default value into a C value +sub FmtDefault { my $name = shift; my $val = shift; - my $literalval = shift; my $type = shift; - if (defined($literalval)) { - die "Both \"default\" and \"defaultliteral\" fields were specified for cvar \"$name\", stopped" if defined($val); - return qq($literalval); - } - - die "Exactly one of \"default\" or \"defaultliteral\" fields must be specified for cvar \"$name\", stopped" unless defined($val); - - if ($type eq "string") { + if ($type eq 'string') { $val =~ s/"/\\"/g; - return qq("$val"); + if ($val eq 'NULL' or $val eq 'null') { return "NULL"; } + else { return qq("$val"); } } - elsif ($type eq "boolean") { + elsif ($type eq 'boolean') { if ($val =~ m/^(0|f(alse)?|no?)$/i) { return qq(0); } elsif ($val =~ m/^(1|t(rue)?|y(es)?)$/i) { return qq(1); } else { @@ -512,11 +490,10 @@ sub fmt_default { return qq(0); # fail-false } } - elsif ($type eq "range") { + elsif ($type eq 'range') { if ($val !~ "-?[0-9]+:-?[0-9]+") { die "Unable to parse range value '$val', stopped"; } - $val =~ s/:/,/; return qq({$val}); } @@ -525,8 +502,8 @@ sub fmt_default { } } -# turns foo_BAR-baz.h into FOO_BAR_BAZ_H_INCLUDED -sub header_to_incl_guard { +# turns /path/foo_BAR-baz.h into FOO_BAR_BAZ_H_INCLUDED +sub Header2InclGuard { my $header_file = shift; my $guard = basename($header_file); $guard =~ tr/a-z\-./A-Z__/; @@ -535,19 +512,7 @@ sub header_to_incl_guard { return $guard; } -sub md5sum { - my $file = shift; - my $md5 = Digest::MD5->new(); - - open FILE, '<', $file; - binmode(FILE); - $md5->addfile(*FILE); - close FILE; - - return $md5->hexdigest; -} - -sub type_to_env_fn { +sub Type2EnvFn { my $type = shift; my %typemap = ( 'int' => 'int', @@ -561,3 +526,84 @@ sub type_to_env_fn { return $typemap{$type}; } +# Parse a file, search the MPI_T_CVAR_INFO_BLOCK if any. +# Distill cvars and categories from the block. +# Push the results to back of @cvars and @categories respectively. +sub ProcessFile { + my $cfile = $_[0]; + my $cvar_info_block = undef; + my $in_cvar_info_block = 0; + + #print "Processing file $cfile\n" if $debug; + open my $CFILE_HANDLE, "< $cfile" or die "Error: open file $cfile -- $!\n"; + while (<$CFILE_HANDLE>) { + if (/END_MPI_T_CVAR_INFO_BLOCK/) { + last; + } elsif ($in_cvar_info_block) { + $cvar_info_block .= $_; + } elsif (/BEGIN_MPI_T_CVAR_INFO_BLOCK/) { + $in_cvar_info_block = 1; + print "Found MPI_T_CVAR_INFO_BLOCK in $cfile\n" if $debug; + } + } + close $CFILE_HANDLE; + + # Do some checking to ensure a correct cvar info block, also + # add file locations to help users' debugging. + if ($cvar_info_block) { + my $info = ($yaml->read_string($cvar_info_block))->[0]; + if (exists $info->{cvars}) { + # Remember location where the cvar is defined. Put that into + # comments of the generated *.h file so that users know where + # to look when meeting compilation errors. + foreach my $cvar (@{$info->{cvars}}) { + $cvar->{location} = $cfile; + die "ERROR: cvar has no name in $cfile\n" unless exists $cvar->{name}; + die "ERROR: cvar $cvar->{name} has no type in $cfile\n" unless exists $cvar->{type}; + die "ERROR: cvar $cvar->{name} has no verbosity in $cfile\n" unless exists $cvar->{verbosity}; + die "ERROR: cvar $cvar->{name} has no scope in $cfile\n" unless exists $cvar->{scope}; + die "ERROR: cvar $cvar->{name} has no class in $cfile\n" unless exists $cvar->{class}; + die "ERROR: cvar $cvar->{name} has no description in $cfile\n" unless exists $cvar->{description}; + } + push (@cvars, @{$info->{cvars}}); + } + + if (exists $info->{categories}) { + # Do the same trick to categories + foreach my $cat (@{$info->{categories}}) { + $cat->{location} = $cfile; + die "ERROR: category has no name in $cfile\n" unless exists $cat->{name}; + die "ERROR: category $cat->{name} has no description in $cfile\n" + unless exists $cat->{description}; + } + push (@categories, @{$info->{categories}}); + } + } +} + +# Search cfiles recursively in the directory passed in. +# Push file names along with their paths to back of @cfiles. +sub ExpandDir { + my $dir = $_[0]; + my @subdirs = (); + my $DIR_HANDLE; + opendir $DIR_HANDLE, "$dir" or die "Error: open directory $dir -- $!\n"; + while (readdir $DIR_HANDLE) { + if (/^\./) { + next; + } elsif (-d "$dir/$_") { + $subdirs[$#subdirs + 1] = "$dir/$_"; + } elsif (/(.*\.[Cchi])(pp){0,1}$/) { + if (!defined($skipfiles{"$dir/$_"})) + { + $cfiles[$#cfiles + 1] = "$dir/$_"; + } + } + } + closedir $DIR_HANDLE; + + # Recursively search subdirs + foreach $dir (@subdirs) { + ExpandDir($dir); + } +} diff --git a/src/include/Makefile.mk b/src/include/Makefile.mk index 6aa9bf2..12f8098 100644 --- a/src/include/Makefile.mk +++ b/src/include/Makefile.mk @@ -66,5 +66,5 @@ noinst_HEADERS += \ src/include/oputil.h \ src/include/mpiinfo.h -src/include/mpich_cvars.h: src/util/cvar/cvars.yml - $(top_srcdir)/maint/gencvars +src/include/mpich_cvars.h: + $(top_srcdir)/maint/extractcvars --dirs="`cat $(top_srcdir)/maint/cvardirs`" diff --git a/src/util/cvar/Makefile.mk b/src/util/cvar/Makefile.mk index bd27c56..fa1543f 100644 --- a/src/util/cvar/Makefile.mk +++ b/src/util/cvar/Makefile.mk @@ -8,10 +8,8 @@ lib_lib@MPILIBNAME@_la_SOURCES += \ src/util/cvar/mpich_cvars.c -dist_noinst_DATA += src/util/cvar/cvars.yml - if MAINTAINER_MODE # normally built by autogen.sh, but this rebuild rule is here -$(top_srcdir)/src/util/cvar/mpich_cvars.c: $(top_srcdir)/src/util/cvar/cvars.yml $(top_srcdir)/maint/gencvars - ( cd $(top_srcdir) && ./maint/gencvars ) +$(top_srcdir)/src/util/cvar/mpich_cvars.c: $(top_srcdir)/maint/extractcvars + ( cd $(top_srcdir) && ./maint/extractcvars --dirs="`cat ./maint/cvardirs`") endif MAINTAINER_MODE http://git.mpich.org/mpich.git/commitdiff/6239007c3a9c415f5ebfc566788f3ca5e… commit 6239007c3a9c415f5ebfc566788f3ca5eb6dd34a Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Wed Dec 18 16:34:31 2013 -0600 Rename script gencvars to extractcvars Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/maint/gencvars b/maint/extractcvars similarity index 100% rename from maint/gencvars rename to maint/extractcvars http://git.mpich.org/mpich.git/commitdiff/c10aaa185e46c42968dbbaed77925f75e… commit c10aaa185e46c42968dbbaed77925f75e820cc80 Author: Junchao Zhang <jczhang(a)mcs.anl.gov> Date: Mon Nov 11 19:00:07 2013 -0600 Delete cvars.yml since it will be split. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/util/cvar/cvars.yml b/src/util/cvar/cvars.yml deleted file mode 100644 index 3d0301d..0000000 --- a/src/util/cvar/cvars.yml +++ /dev/null @@ -1,770 +0,0 @@ ---- #YAML:1.0 -# This is a YAML formatted file with '#'-style comments. This file -# *must* be parseable by the YAML::Tiny perl module. So don't use -# the inline list or map forms and don't try to use any really fancy -# features like tagging or anchors. -# -# It is used to generate cvar-handling code for MPICH. -# See the maint/gencvars script for more information. -# -# The only tricky bit is that long reflowed text (such as a description) -# should use ">-" to both fold newlines and remove the trailing newline. -# -# Listing "FOO" as a cvar name will cause two environment variables -# to be examined by default: MPIR_CVAR_FOO and MPICH_FOO, with the -# MPIR_PARAM value having priority. -# -# Each cvar has a class associated with it. The following -# classes are defined: "device", and "none". Setting it to "device" -# will allow the device to override the default value. - -categories: - - name : COLLECTIVE - description : cvars that control collective communication behavior - - name : COMMUNICATOR - description : cvars that control communicator construction and operation - - name : PT2PT - description : cvars that control point-to-point communication behavior - - name : RMA - description : cvars that control RMA communicatin behavior - - name : INTRANODE - description : intranode communication cvars - - name : DEVELOPER - description : useful for developers working on MPICH itself - - name : MEMORY - description : affects memory allocation and usage, including MPI object handles - - name : ERROR_HANDLING - description : cvars that control error handling behavior (stack traces, aborts, etc) - - name : DEBUGGER - description : cvars relevant to the "MPIR" debugger interface - - name : CHECKPOINTING - description : cvars relevant to checkpointing - - name : FAULT_TOLERANCE - description : cvars that control fault tolerance behavior - - name : THREADS - description : multi-threading cvars - - name : CH3 - description : cvars that control behavior of ch3 - - name : NEMESIS - description : "cvars that control behavior of the ch3:nemesis channel" - - name : SOCKETS - description : control socket cvars - - name : PROCESS_MANAGER - description : cvars that control the client-side process manager code - -cvars: - ############################################################## - # collective cvars - - category : COLLECTIVE - name : ALLTOALL_SHORT_MSG_SIZE - type : int - default : 256 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the short message algorithm will be used if the per-destination - message size (sendcount*size(sendtype)) is <= this value - - - category : COLLECTIVE - name : ALLTOALL_MEDIUM_MSG_SIZE - type : int - default : 32768 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the medium message algorithm will be used if the per-destination - message size (sendcount*size(sendtype)) is <= this value and - larger than ALLTOALL_SHORT_MSG_SIZE - - - category : COLLECTIVE - name : ALLTOALL_THROTTLE - type : int - default : 32 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - max no. of irecvs/isends posted at a time in some alltoall - algorithms. Setting it to 0 causes all irecvs/isends to be - posted at once. - - - category : COLLECTIVE - name : REDSCAT_COMMUTATIVE_LONG_MSG_SIZE - type : int - default : 524288 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the long message algorithm will be used if the operation is commutative - and the send buffer size is >= this value (in bytes) - - - category : COLLECTIVE - name : BCAST_MIN_PROCS - type : int - default : 8 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the minimum number of processes in a communicator to use a non-binomial - broadcast algorithm - - - category : COLLECTIVE - name : BCAST_SHORT_MSG_SIZE - type : int - default : 12288 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the short message algorithm will be used if the send buffer size is < - this value (in bytes) - - - category : COLLECTIVE - name : BCAST_LONG_MSG_SIZE - type : int - default : 524288 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the long message algorithm will be used if the send buffer size is >= - this value (in bytes) - - - category : COLLECTIVE - name : ALLGATHER_SHORT_MSG_SIZE - type : int - default : 81920 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - For MPI_Allgather and MPI_Allgatherv, the short message algorithm will - be used if the send buffer size is < this value (in bytes). - - - category : COLLECTIVE - name : ALLGATHER_LONG_MSG_SIZE - type : int - default : 524288 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - For MPI_Allgather and MPI_Allgatherv, the long message algorithm will be - used if the send buffer size is >= this value (in bytes) - - - category : COLLECTIVE - name : REDUCE_SHORT_MSG_SIZE - type : int - default : 2048 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the short message algorithm will be used if the send buffer size is <= - this value (in bytes) - - - category : COLLECTIVE - name : ALLREDUCE_SHORT_MSG_SIZE - type : int - default : 2048 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - the short message algorithm will be used if the send buffer size is <= - this value (in bytes) - - - category : COLLECTIVE - name : GATHER_VSMALL_MSG_SIZE - type : int - default : 1024 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - use a temporary buffer for intracommunicator MPI_Gather if the send - buffer size is < this value (in bytes) - - - category : COLLECTIVE - name : GATHER_INTER_SHORT_MSG_SIZE - type : int - default : 2048 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - use the short message algorithm for intercommunicator MPI_Gather if the - send buffer size is < this value (in bytes) - - - category : COLLECTIVE - name : GATHERV_INTER_SSEND_MIN_PROCS - type : int - default : 32 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Use Ssend (synchronous send) for intercommunicator MPI_Gatherv if the - "group B" size is >= this value. Specifying "-1" always avoids using - Ssend. For backwards compatibility, specifying "0" uses the default - value. - - - category : COLLECTIVE - name : SCATTER_INTER_SHORT_MSG_SIZE - type : int - default : 2048 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - use the short message algorithm for intercommunicator MPI_Scatter if the - send buffer size is < this value (in bytes) - - - category : COLLECTIVE - name : ALLGATHERV_PIPELINE_MSG_SIZE - type : int - default : 32768 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - The smallest message size that will be used for the pipelined, large-message, - ring algorithm in the MPI_Allgatherv implementation. - - - category : COLLECTIVE - name : ENABLE_SMP_COLLECTIVES - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Enable SMP aware collective communication. - - - category : COLLECTIVE - name : ENABLE_SMP_BARRIER - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Enable SMP aware barrier. - - - category : COLLECTIVE - name : ENABLE_SMP_BCAST - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Enable SMP aware broadcast. - - - category : COLLECTIVE - name : MAX_SMP_BCAST_MSG_SIZE - type : int - default : 0 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Maximum message size for which SMP-aware broadcast is used. A - value of '0' uses SMP-aware broadcast for all message sizes. - - - category : COLLECTIVE - name : ENABLE_SMP_REDUCE - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Enable SMP aware reduce. - - - category : COLLECTIVE - name : MAX_SMP_REDUCE_MSG_SIZE - type : int - default : 0 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Maximum message size for which SMP-aware reduce is used. A - value of '0' uses SMP-aware reduce for all message sizes. - - - category : COLLECTIVE - name : ENABLE_SMP_ALLREDUCE - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Enable SMP aware allreduce. - - - category : COLLECTIVE - name : MAX_SMP_ALLREDUCE_MSG_SIZE - type : int - default : 0 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Maximum message size for which SMP-aware allreduce is used. A - value of '0' uses SMP-aware allreduce for all message sizes. - - ############################################################## - # communicator cvars - - category : COMMUNICATOR - name : COMM_SPLIT_USE_QSORT - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Use qsort(3) in the implementation of MPI_Comm_split instead of bubble sort. - - ############################################################## - # debugging cvars - - category : DEBUGGER - name : DEBUG_HOLD - type : boolean - default : false - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, causes processes to wait in MPI_Init and - MPI_Initthread for a debugger to be attached. Once the - debugger has attached, the variable 'hold' should be set to 0 - in order to allow the process to continue (e.g., in gdb, "set - hold=0"). - - - category : DEBUGGER - name : PROCTABLE_SIZE - type : int - default : 64 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Size of the "MPIR" debugger interface proctable (process table). - - - category : DEBUGGER - name : PROCTABLE_PRINT - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, dump the proctable entries at MPIR_WaitForDebugger-time. - (currently compile-time disabled by "#if 0") - - ############################################################## - # fault-tolerance cvars - - category : FAULT_TOLERANCE - name : ENABLE_COLL_FT_RET - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - DEPRECATED! Will be removed in MPICH-3.2 - Collectives called on a communicator with a failed process - should not hang, however the result of the operation may be - invalid even though the function returns MPI_SUCCESS. This - option enables an experimental feature that will return an error - if the result of the collective is invalid. - - ############################################################## - # memory cvars - - category : MEMORY - name : ABORT_ON_LEAKED_HANDLES - type : boolean - default : false - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, MPI will call MPI_Abort at MPI_Finalize if any MPI object - handles have been leaked. For example, if MPI_Comm_dup is called - without calling a corresponding MPI_Comm_free. For uninteresting - reasons, enabling this option may prevent all known object leaks from - being reported. MPICH must have been configure with - "--enable-g=handlealloc" or better in order for this functionality to - work. - - ############################ - # threads cvars - - category : THREADS - name : CTXID_EAGER_SIZE - type : int - default : 2 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - The MPIR_CVAR_CTXID_EAGER_SIZE environment variable allows you to - specify how many words in the context ID mask will be set aside - for the eager allocation protocol. If the application is running - out of context IDs, reducing this value may help. - - - category : THREADS - name : ASYNC_PROGRESS - type : boolean - default : false - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If set to true, MPICH will initiate an additional thread to - make asynchronous progress on all communication operations - including point-to-point, collective, one-sided operations and - I/O. Setting this variable will automatically increase the - thread-safety level to MPI_THREAD_MULTIPLE. While this - improves the progress semantics, it might cause a small amount - of performance overhead for regular MPI operations. The user - is encouraged to leave one or more hardware threads vacant in - order to prevent contention between the application threads - and the progress thread(s). The impact of oversubscription is - highly system dependent but may be substantial in some cases, - hence this recommendation. - - - category : THREADS - name : DEFAULT_THREAD_LEVEL - type : string - default : "MPI_THREAD_SINGLE" - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Sets the default thread level to use when using MPI_INIT. - - ############################################################## - # other MPI-level cvars - - category : DEVELOPER - name : MEMDUMP - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_MPIDEV_DETAIL - scope : MPI_T_SCOPE_LOCAL - description : >- - If true, list any memory that was allocated by MPICH and that - remains allocated when MPI_Finalize completes. - - - category : ERROR_HANDLING - name : ERROR_CHECKING - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, perform checks for errors, typically to verify valid inputs - to MPI routines. Only effective when MPICH is configured with - --enable-error-checking=runtime . - - - category : ERROR_HANDLING - name : PRINT_ERROR_STACK - type : boolean - default : true - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, print an error stack trace at error handling time. - - - category : ERROR_HANDLING - name : CHOP_ERROR_STACK - type : int - default : 0 - class : device - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If >0, truncate error stack output lines this many characters - wide. If 0, do not truncate, and if <0 use a sensible default. - - ############################################################## - # ch3 cvars - - category : CH3 - name : CH3_RMA_ACC_IMMED - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Use the immediate accumulate optimization - - - category : CH3 - name : CH3_RMA_NREQUEST_THRESHOLD - type : int - default : 4000 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Threshold at which the RMA implementation attempts to complete requests - while completing RMA operations and while using the lazy synchonization - approach. Change this value if programs fail because they run out of - requests or other internal resources - - - category : CH3 - name : CH3_RMA_NREQUEST_NEW_THRESHOLD - type : int - default : 128 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Threshold for the number of new requests since the last attempt to - complete pending requests. Higher values can increase performance, - but may run the risk of exceeding the available number of requests - or other internal resources. - - - category : CH3 - name : CH3_RMA_LOCK_IMMED - type : boolean - default : false - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Issue a request for the passive target RMA lock immediately. Default - behavior is to defer the lock request until the call to MPI_Win_unlock. - - - category : CH3 - name : CH3_RMA_MERGE_LOCK_OP_UNLOCK - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Enable/disable an optimization that merges lock, op, and unlock - messages, for single-operation passive target epochs. - - - category : CH3 - name : CH3_NOLOCAL - alt-env : - - CH3_NO_LOCAL - type : boolean - default : false - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, force all processes to operate as though all processes - are located on another node. For example, this disables shared - memory communication hierarchical collectives. - - - category : CH3 - name : CH3_ODD_EVEN_CLIQUES - alt-env : - - CH3_EVEN_ODD_CLIQUES - type : boolean - default : false - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, odd procs on a node are seen as local to each other, and even - procs on a node are seen as local to each other. Used for debugging on - a single machine. - - - category : CH3 - name : CH3_INTERFACE_HOSTNAME - alt-env : - - INTERFACE_HOSTNAME - type : string - defaultliteral : NULL - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If non-NULL, this cvar specifies the IP address that - other processes should use when connecting to this process. - This cvar is mutually exclusive with the - MPIR_CVAR_CH3_NETWORK_IFACE cvar and it is an error to set them - both. - - - category : CH3 - name : CH3_EAGER_MAX_MSG_SIZE - type : int - default : 131072 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This cvar controls the message size at which CH3 switches - from eager to rendezvous mode. - - - category : CH3 - name : CH3_PORT_RANGE - alt-env : - - PORTRANGE - - PORT_RANGE - type : range - default : "0:0" - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - The MPIR_CVAR_CH3_PORT_RANGE environment variable allows you to - specify the range of TCP ports to be used by the process - manager and the MPICH library. The format of this variable is - <low>:<high>. To specify any available port, use 0:0. - - ############################################################## - # nemesis cvars - - category : NEMESIS - name : NEMESIS_POLLS_BEFORE_YIELD - type : int - default : 1000 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - When MPICH is in a busy waiting loop, it will periodically - call a function to yield the processor. This cvar sets - the number of loops before the yield function is called. A - value of 0 disables yielding. - - - category : NEMESIS - name : NEMESIS_LMT_DMA_THRESHOLD - type : int - default : 2097152 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Messages larger than this size will use the "dma" (knem) - intranode LMT implementation, if it is enabled and available. - - - category : NEMESIS - name : NEMESIS_NETMOD - type : string - default : "" - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If non-empty, this cvar specifies which network module - should be used for communication. - - - category : NEMESIS - name : NEMESIS_SHM_EAGER_MAX_SZ - type : int - default : -1 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This cvar controls the message size at which Nemesis - switches from eager to rendezvous mode for shared memory. - If this cvar is set to -1, then Nemesis will choose - an appropriate value. - - - category : NEMESIS - name : NEMESIS_SHM_READY_EAGER_MAX_SZ - type : int - default : -2 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This cvar controls the message size at which Nemesis - switches from eager to rendezvous mode for ready-send - messages. If this cvar is set to -1, then ready messages - will always be sent eagerly. If this cvar is set to -2, - then Nemesis will choose an appropriate value. - - - category : NEMESIS - name : NEMESIS_ENABLE_CKPOINT - type : boolean - default : false - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If true, enables checkpointing support and returns an error if - checkpointing library cannot be initialized. - - ############################################################## - # nemesis TCP cvars - - category : NEMESIS - name : NEMESIS_TCP_NETWORK_IFACE - alt-env : - - NETWORK_IFACE - type : string - defaultliteral : NULL - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If non-NULL, this cvar specifies which pseudo-ethernet - interface the tcp netmod should use (e.g., "eth1", "ib0"). - Note, this is a Linux-specific cvar. - This cvar is mutually exclusive with the - MPIR_CVAR_CH3_INTERFACE_HOSTNAME cvar and it is an error to set - them both. - - - category : NEMESIS - name : NEMESIS_TCP_HOST_LOOKUP_RETRIES - type : int - default : 10 - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This cvar controls the number of times to retry the - gethostbyname() function before giving up. - - ############################################################## - # nemesis portals cvars - - category : NEMESIS - name : NEMESIS_PORTALS_COMM_OVERRIDES - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - If set to false, communication override functionality will be - disabled for netmods that provide the override feature. - # Note the netmod must implement this functionality by not - # setting the comm_ops table in any VC. - - ############################################################## - # process manager cvars - - category : PROCESS_MANAGER - name : NAMESERV_FILE_PUBDIR - alt-env : - - NAMEPUB_DIR - type : string - defaultliteral : NULL - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - Sets the directory to use for MPI service publishing in the - file nameserv implementation. Allows the user to override - where the publish and lookup information is placed for - connect/accept based applications. - -... - -# Local Variables: -# mode: conf-colon -# indent-tabs-mode: nil -# End: ----------------------------------------------------------------------- Summary of changes: autogen.sh | 10 +- maint/Makefile.mk | 3 +- maint/{errmsgdirs => cvardirs} | 0 maint/extractcvars | 609 ++++++++++++++++ maint/gencvars | 563 -------------- src/include/Makefile.mk | 4 +- src/include/mpiutil.h | 20 + src/mpi/coll/allgather.c | 29 + src/mpi/coll/allgatherv.c | 18 + src/mpi/coll/allreduce.c | 49 ++ src/mpi/coll/alltoall.c | 46 ++ src/mpi/coll/barrier.c | 16 + src/mpi/coll/bcast.c | 86 +++ src/mpi/coll/gather.c | 29 + src/mpi/coll/gatherv.c | 22 +- src/mpi/coll/helper_fns.c | 26 + src/mpi/coll/red_scat.c | 18 + src/mpi/coll/reduce.c | 40 + src/mpi/coll/scatter.c | 18 + src/mpi/comm/comm_split.c | 21 + src/mpi/comm/commutil.c | 21 + src/mpi/debugger/dbginit.c | 29 +- src/mpi/errhan/errutil.c | 32 + src/mpi/init/finalize.c | 22 + src/mpi/init/init.c | 41 + src/mpi/init/initthread.c | 37 + .../ch3/channels/nemesis/netmod/tcp/tcp_init.c | 63 ++ src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c | 17 + src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c | 38 + .../ch3/channels/nemesis/src/mpid_nem_lmt_dma.c | 18 + .../ch3/channels/nemesis/src/mpid_nem_network.c | 18 + src/mpid/ch3/src/ch3u_rma_sync.c | 69 ++ src/mpid/ch3/src/mpid_vc.c | 43 ++ src/nameserv/file/file_nameserv.c | 25 + src/util/cvar/Makefile.mk | 6 +- src/util/cvar/cvars.yml | 770 -------------------- src/util/mem/handlemem.c | 27 + 37 files changed, 1555 insertions(+), 1348 deletions(-) copy maint/{errmsgdirs => cvardirs} (100%) create mode 100755 maint/extractcvars delete mode 100755 maint/gencvars delete mode 100644 src/util/cvar/cvars.yml hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-90-gb2b9e1f
by noreply＠mpich.org 19 Dec '13

19 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via b2b9e1f9a5bbaf79126867531a7c685835319a04 (commit) from 37012b672626a5113b2d5707d8c8af6a850de71e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/b2b9e1f9a5bbaf79126867531a7c68583… commit b2b9e1f9a5bbaf79126867531a7c685835319a04 Author: Pavan Balaji <balaji(a)mcs.anl.gov> Date: Thu Dec 19 13:44:23 2013 +0800 White space cleanup. Signed-off-by: Masamichi Takagi <masamichi.takagi(a)gmail.com> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index 3fae51f..0948cda 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -36,9 +36,9 @@ static int ib_initialized = 0; static int maxcon; static struct ibv_device **ib_devlist; static struct ibv_context *ib_ctx; -struct ibv_context *MPID_nem_ib_ctx_export; /* for SC13 demo connector */ +struct ibv_context *MPID_nem_ib_ctx_export; /* for SC13 demo connector */ static struct ibv_pd *ib_pd; -struct ibv_pd *MPID_nem_ib_pd_export; /* for SC13 demo connector */ +struct ibv_pd *MPID_nem_ib_pd_export; /* for SC13 demo connector */ struct ibv_cq *MPID_nem_ib_rc_shared_scq; struct ibv_cq *MPID_nem_ib_rc_shared_scq_lmt_put; struct ibv_cq *MPID_nem_ib_rc_shared_scq_scratch_pad; @@ -210,7 +210,8 @@ static int MPID_nem_ib_com_device_init() else { ib_ctx = ibv_open_device(ib_devlist[i]); } - dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_ctx_export=%p,ib_ctx=%p\n", MPID_nem_ib_ctx_export, ib_ctx); + dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_ctx_export=%p,ib_ctx=%p\n", + MPID_nem_ib_ctx_export, ib_ctx); if (!ib_ctx) { fprintf(stderr, "failed to open IB device\n"); goto err_exit; @@ -228,7 +229,8 @@ static int MPID_nem_ib_com_device_init() else { ib_pd = ibv_alloc_pd(ib_ctx); } - dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_pd_export=%p,ib_pd=%p\n", MPID_nem_ib_pd_export, ib_pd); + dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_pd_export=%p,ib_pd=%p\n", + MPID_nem_ib_pd_export, ib_pd); if (!ib_pd) { fprintf(stderr, "ibv_alloc_pd failed\n"); goto err_exit; @@ -323,7 +325,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) int open_flag_conn = open_flag; if (open_flag_conn != MPID_NEM_IB_COM_OPEN_RC && open_flag_conn != MPID_NEM_IB_COM_OPEN_RC_LMT_PUT && - open_flag_conn != MPID_NEM_IB_COM_OPEN_UD && open_flag_conn != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD) { + open_flag_conn != MPID_NEM_IB_COM_OPEN_UD && + open_flag_conn != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD) { dprintf("MPID_nem_ib_com_open,bad flag\n"); ibcom_errno = -1; goto fn_fail; @@ -383,9 +386,11 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) #ifdef HAVE_LIBDCFA MPID_nem_ib_rc_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_rc_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_rc_shared_scq = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq, -1, dprintf("MPID_nem_ib_rc_shared_scq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq, -1, + dprintf("MPID_nem_ib_rc_shared_scq")); } conp->icom_scq = MPID_nem_ib_rc_shared_scq; @@ -393,51 +398,65 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) #ifdef HAVE_LIBDCFA MPID_nem_ib_rc_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_rc_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_rc_shared_rcq = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq, -1, dprintf("MPID_nem_ib_rc_shared_rcq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq, -1, + dprintf("MPID_nem_ib_rc_shared_rcq")); } conp->icom_rcq = MPID_nem_ib_rc_shared_rcq; break; case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: if (!MPID_nem_ib_rc_shared_scq_scratch_pad) { #ifdef HAVE_LIBDCFA - MPID_nem_ib_rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); + MPID_nem_ib_rc_shared_scq_scratch_pad = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_rc_shared_scq_scratch_pad = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_scratch_pad, -1, dprintf("MPID_nem_ib_rc_shared_scq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_scratch_pad, -1, + dprintf("MPID_nem_ib_rc_shared_scq")); } conp->icom_scq = MPID_nem_ib_rc_shared_scq_scratch_pad; if (!MPID_nem_ib_rc_shared_rcq_scratch_pad) { #ifdef HAVE_LIBDCFA - MPID_nem_ib_rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); + MPID_nem_ib_rc_shared_rcq_scratch_pad = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_rc_shared_rcq_scratch_pad = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_scratch_pad, -1, dprintf("MPID_nem_ib_rc_shared_rcq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_scratch_pad, -1, + dprintf("MPID_nem_ib_rc_shared_rcq")); } conp->icom_rcq = MPID_nem_ib_rc_shared_rcq_scratch_pad; break; case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: if (!MPID_nem_ib_rc_shared_scq_lmt_put) { #ifdef HAVE_LIBDCFA - MPID_nem_ib_rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); + MPID_nem_ib_rc_shared_scq_lmt_put = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_rc_shared_scq_lmt_put = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_lmt_put, -1, dprintf("MPID_nem_ib_rc_shared_scq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_lmt_put, -1, + dprintf("MPID_nem_ib_rc_shared_scq")); } conp->icom_scq = MPID_nem_ib_rc_shared_scq_lmt_put; if (!MPID_nem_ib_rc_shared_rcq_lmt_put) { #ifdef HAVE_LIBDCFA - MPID_nem_ib_rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); + MPID_nem_ib_rc_shared_rcq_lmt_put = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_rc_shared_rcq_lmt_put = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_lmt_put, -1, dprintf("MPID_nem_ib_rc_shared_rcq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_lmt_put, -1, + dprintf("MPID_nem_ib_rc_shared_rcq")); } conp->icom_rcq = MPID_nem_ib_rc_shared_rcq_lmt_put; break; @@ -446,9 +465,11 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) #ifdef HAVE_LIBDCFA MPID_nem_ib_ud_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_ud_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_ud_shared_scq = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_scq, -1, dprintf("MPID_nem_ib_ud_shared_scq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_scq, -1, + dprintf("MPID_nem_ib_ud_shared_scq")); } conp->icom_scq = MPID_nem_ib_ud_shared_scq; @@ -456,9 +477,11 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) #ifdef HAVE_LIBDCFA MPID_nem_ib_ud_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); #else - MPID_nem_ib_ud_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); + MPID_nem_ib_ud_shared_rcq = + ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); #endif - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_rcq, -1, dprintf("MPID_nem_ib_ud_shared_rcq")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_rcq, -1, + dprintf("MPID_nem_ib_ud_shared_rcq")); } conp->icom_rcq = MPID_nem_ib_ud_shared_rcq; break; @@ -538,17 +561,20 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) /* RDMA-write-from local memory area */ conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM] = MPID_NEM_IB_COM_RDMABUF_SZ; conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] = - mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ); + mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", + conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ); if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] == (void *) -1) { fprintf(stderr, "failed to allocate buffer\n"); goto err_exit; } - memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], 0, conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]); + memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], 0, + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]); conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM] = MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], - conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]); + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]); if (!conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]) { fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); goto err_exit; @@ -568,28 +594,34 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) } #else conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] = - mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], MPID_NEM_IB_COM_RDMABUF_SZ); + mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], + MPID_NEM_IB_COM_RDMABUF_SZ); #endif if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] == (void *) -1) { fprintf(stderr, "failed to allocate buffer\n"); goto err_exit; } - memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], 0, conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO]); + memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], 0, + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO]); conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO] = - ibv_reg_mr(ib_pd, conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO], - mr_flags); + ibv_reg_mr(ib_pd, conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO], mr_flags); if (!conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]) { fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); goto err_exit; } #ifdef HAVE_LIBDCFA - dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->buf); + dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->buf); #else - dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->addr); + dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->addr); #endif - dprintf("MPID_nem_ib_com_open,fd=%d,rkey=%08x\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->rkey); + dprintf("MPID_nem_ib_com_open,fd=%d,rkey=%08x\n", *condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->rkey); /* RDMA-write-to remote memory area */ conp->icom_rmem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); @@ -644,40 +676,51 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; /* UD-write-from local memory area */ - MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, + dprintf("buf_size too short\n")); conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM] = MPID_NEM_IB_COM_UDBUF_SZ; conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] = - mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], MPID_NEM_IB_COM_UDBUF_SZ); + mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], + MPID_NEM_IB_COM_UDBUF_SZ); MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] == (void *) -1, -1, - dprintf("failed to allocate buffer\n")); - memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], 0, conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]); + dprintf("failed to allocate buffer\n")); + memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], 0, + conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]); conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM] = - MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]); + MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], + conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); /* UD-write-to local memory area */ /* addr to addr+39 are not filled, addr+40 to addr+length-1 are filled with payload */ - MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, + dprintf("buf_size too short\n")); conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO] = MPID_NEM_IB_COM_UDBUF_SZ; conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO] = - mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], MPID_NEM_IB_COM_UDBUF_SZ); + mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], + MPID_NEM_IB_COM_UDBUF_SZ); MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO] == (void *) -1, -1, - dprintf("failed to allocate buffer\n")); - memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], 0, conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]); + dprintf("failed to allocate buffer\n")); + memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], 0, + conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]); conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO] = - MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]); + MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], + conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); /* initialize arena allocator for MPID_NEM_IB_COM_UDWR_TO */ //MPID_nem_ib_com_udbuf_init(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO]); - dprintf("MPID_nem_ib_com_open,ud,fd=%d,lkey=%08x\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO]->lkey); + dprintf("MPID_nem_ib_com_open,ud,fd=%d,lkey=%08x\n", *condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO]->lkey); break; default: MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, dprintf("invalid open_flag\n")); @@ -691,7 +734,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) /* SR (send request) template */ conp->icom_sr = - (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_NTEMPLATE); + (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * + MPID_NEM_IB_COM_RC_SR_NTEMPLATE); memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_NTEMPLATE); for (i = 0; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { @@ -701,13 +745,14 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) sizeof(struct ibv_sge) * WR_SG_NUM); #else sge = - (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE); + (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * + MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE); #endif conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].next = (i == - MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) ? NULL : &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + - i + 1]; + MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - + 1) ? NULL : &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i + 1]; #ifdef HAVE_LIBDCFA #else conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list = sge; @@ -742,7 +787,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) sizeof(struct ibv_sge) * WR_SG_NUM); #else sge = - (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_INITIATOR_NSGE); + (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * + MPID_NEM_IB_COM_LMT_INITIATOR_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_INITIATOR_NSGE); #endif conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].next = NULL; @@ -756,7 +802,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) /* SR (send request) template for MPID_NEM_IB_COM_LMT_PUT *//* for lmt-put-done */ #ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); + memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, + sizeof(struct ibv_sge) * WR_SG_NUM); #else sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); @@ -777,7 +824,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) /* RR (receive request) template for MPID_NEM_IB_COM_RDMAWR_RESPONDER */ conp->icom_rr = - (struct ibv_recv_wr *) malloc(sizeof(struct ibv_recv_wr) * MPID_NEM_IB_COM_RC_RR_NTEMPLATE); + (struct ibv_recv_wr *) malloc(sizeof(struct ibv_recv_wr) * + MPID_NEM_IB_COM_RC_RR_NTEMPLATE); memset(conp->icom_rr, 0, sizeof(struct ibv_recv_wr) * MPID_NEM_IB_COM_RC_RR_NTEMPLATE); /* create one dummy RR to ibv_post_recv */ @@ -794,7 +842,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) conp->icom_sr = (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE); - memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE); + memset(conp->icom_sr, 0, + sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE); /* SR (send request) template for MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR */ #ifdef HAVE_LIBDCFA @@ -822,10 +871,12 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) conp->icom_sr = (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE); - memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE); + memset(conp->icom_sr, 0, + sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE); /* SR (send request) template for MPID_NEM_IB_COM_LMT_PUT */ #ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); + memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, + sizeof(struct ibv_sge) * WR_SG_NUM); #else sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); @@ -854,25 +905,27 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) conp->icom_ah_attr = - (struct ibv_ah_attr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, sizeof(struct ibv_ah_attr)); + (struct ibv_ah_attr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, + sizeof(struct ibv_ah_attr)); conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].sl = 0; conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].src_path_bits = 0; - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].static_rate = 0; /* not limit on static rate (100% port speed) */ + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].static_rate = 0; /* not limit on static rate (100% port speed) */ conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 0; conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].port_num = conp->icom_port; #if 0 conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 1; conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.flow_label = 0; - conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */ + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */ conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.hop_limit = 1; conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.traffic_class = 0; #endif /* SR (send request) template for MPID_NEM_IB_COM_UD_INITIATOR */ conp->icom_sr = - (struct ibv_send_wr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, sizeof(struct ibv_send_wr)); + (struct ibv_send_wr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, + sizeof(struct ibv_send_wr)); conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].next = NULL; #ifdef HAVE_LIBDCFA @@ -898,7 +951,8 @@ int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) /* RR (receive request) template for MPID_NEM_IB_COM_UD_RESPONDER */ conp->icom_rr = - (struct ibv_recv_wr *) calloc(MPID_NEM_IB_COM_UD_RR_NTEMPLATE, sizeof(struct ibv_recv_wr)); + (struct ibv_recv_wr *) calloc(MPID_NEM_IB_COM_UD_RR_NTEMPLATE, + sizeof(struct ibv_recv_wr)); /* create one dummy RR to ibv_post_recv */ conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].next = NULL; @@ -941,7 +995,7 @@ int MPID_nem_ib_com_alloc(int condesc, int sz) scratch_pad = mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); dprintf("MPID_nem_ib_com_alloc,mmap=%p,len=%d\n", scratch_pad, sz); MPID_NEM_IB_COM_ERR_CHKANDJUMP(scratch_pad == (void *) -1, -1, - dprintf("failed to allocate buffer\n")); + dprintf("failed to allocate buffer\n")); dprintf("MPID_nem_ib_com_alloc,scratch_pad=%p\n", scratch_pad); memset(scratch_pad, 0, sz); } @@ -950,9 +1004,9 @@ int MPID_nem_ib_com_alloc(int condesc, int sz) conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO] = MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO], - conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_TO]); + conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_TO]); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); #ifdef HAVE_LIBDCFA dprintf("MPID_nem_ib_com_alloc,fd=%d,rmem=%p\n", condesc, @@ -966,7 +1020,8 @@ int MPID_nem_ib_com_alloc(int condesc, int sz) break; default: MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, - dprintf("MPID_nem_ib_com_alloc, invalid open_flag=%d\n", conp->open_flag)); + dprintf("MPID_nem_ib_com_alloc, invalid open_flag=%d\n", + conp->open_flag)); break; } @@ -993,7 +1048,8 @@ int MPID_nem_ib_com_close(int condesc) goto fn_exit; } -int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid) +int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, + union ibv_gid *remote_gid) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1073,8 +1129,8 @@ int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, unio } #define MPID_NEM_IB_ENABLE_INLINE -int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, int sz_hdr, - void *data, int sz_data, int *copied) +int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, + int sz_hdr, void *data, int sz_data, int *copied) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1082,8 +1138,8 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref int ib_errno; int num_sge; - dprintf("MPID_nem_ib_com_isend,prefix=%p,sz_prefix=%d,hdr=%p,sz_hdr=%d,data=%p,sz_data=%d\n", prefix, - sz_prefix, hdr, sz_hdr, data, sz_data); + dprintf("MPID_nem_ib_com_isend,prefix=%p,sz_prefix=%d,hdr=%p,sz_hdr=%d,data=%p,sz_data=%d\n", + prefix, sz_prefix, hdr, sz_hdr, data, sz_data); MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); if (conp->icom_connected == 0) { @@ -1103,7 +1159,9 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf_from; - sz_hdrmagic->sz = sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t); + sz_hdrmagic->sz = + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data + + sizeof(MPID_nem_ib_tailmagic_t); sz_hdrmagic->magic = MPID_NEM_IB_COM_MAGIC; /* memcpy hdr is needed because hdr resides in stack when sending close-VC command */ @@ -1115,8 +1173,9 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) sz_hdrmagic; conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) sz_hdrmagic - - (uint64_t) conp-> - icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); + (uint64_t) + conp->icom_mem + [MPID_NEM_IB_COM_RDMAWR_FROM]); #else conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) sz_hdrmagic; #endif @@ -1129,7 +1188,8 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref if (sz_data) { //dprintf("MPID_nem_ib_com_isend,data=%p,sz_data=%d\n", data, sz_data); struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, + printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); #ifdef HAVE_LIBDCFA conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) data; conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = @@ -1144,27 +1204,31 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref int sz_pad = sz_data_pow2 - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data); MPID_nem_ib_tailmagic_t *tailmagic = - (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad); + (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + + sz_prefix + sz_hdr + sz_pad); tailmagic->magic = MPID_NEM_IB_COM_MAGIC; #ifdef HAVE_LIBDCFA conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr; conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from + - sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + - sz_hdr - - (uint64_t) conp-> - icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); + sizeof + (MPID_nem_ib_sz_hdrmagic_t) + + sz_prefix + sz_hdr - + (uint64_t) + conp->icom_mem + [MPID_NEM_IB_COM_RDMAWR_FROM]); #else conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr; #endif - conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length = sz_pad + sizeof(MPID_nem_ib_tailmagic_t); + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length = + sz_pad + sizeof(MPID_nem_ib_tailmagic_t); conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].lkey = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; num_sge += 1; - dprintf("MPID_nem_ib_com_isend,sz_data=%d,pow2=%d,sz_pad=%d,num_sge=%d\n", sz_data, sz_data_pow2, sz_pad, - num_sge); + dprintf("MPID_nem_ib_com_isend,sz_data=%d,pow2=%d,sz_pad=%d,num_sge=%d\n", sz_data, + sz_data_pow2, sz_pad, num_sge); conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].num_sge = num_sge; conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id = wr_id; @@ -1193,12 +1257,15 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref #endif #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE]); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", ib_errno)); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", + ib_errno)); #else ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE], &bad_wr); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); + dprintf + ("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); #endif #ifdef MPID_NEM_IB_ENABLE_INLINE if (sumsz <= conp->max_inline_data) { @@ -1221,7 +1288,8 @@ int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_pref goto fn_exit; } -int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, int sz_data) +int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, + int sz_data) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1230,13 +1298,14 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h int sz_data_rem = sz_data; int i; struct ibv_mr *mr_data; - uint32_t sumsz = sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t); + uint32_t sumsz = + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t); unsigned long tscs, tsce; dprintf("MPID_nem_ib_com_isend_chain,enter\n"); MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_connected == 0, -1, - printf("MPID_nem_ib_com_isend_chain,icom_connected==0\n")); + printf("MPID_nem_ib_com_isend_chain,icom_connected==0\n")); void *buf_from = (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] + @@ -1248,12 +1317,17 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h /* let the last command icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAIN-1] which has IBV_WR_RDMA_WRITE_WITH_IMM */ int s = - MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data_pow2 + - sizeof(MPID_nem_ib_tailmagic_t) + MPID_NEM_IB_COM_INLINE_DATA - 1) / MPID_NEM_IB_COM_INLINE_DATA; - MPID_NEM_IB_COM_ERR_CHKANDJUMP((sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data_pow2) % 4 != 0, -1, - printf("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); + MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + + sz_data_pow2 + sizeof(MPID_nem_ib_tailmagic_t) + + MPID_NEM_IB_COM_INLINE_DATA - + 1) / MPID_NEM_IB_COM_INLINE_DATA; + MPID_NEM_IB_COM_ERR_CHKANDJUMP((sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + + sz_data_pow2) % 4 != 0, -1, + printf + ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); MPID_NEM_IB_COM_ERR_CHKANDJUMP(s < 0 || - s >= MPID_NEM_IB_COM_SMT_INLINE_NCHAIN, -1, printf("MPID_nem_ib_com_isend_chain,s\n")); + s >= MPID_NEM_IB_COM_SMT_INLINE_NCHAIN, -1, + printf("MPID_nem_ib_com_isend_chain,s\n")); dprintf("MPID_nem_ib_com_isend_chain,sz_hdr=%d,sz_data=%d,s=%d\n", sz_hdr, sz_data, s); for (i = s; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { @@ -1271,8 +1345,9 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h (uint64_t) buf_from; conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) conp-> - icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); + (uint64_t) + conp->icom_mem + [MPID_NEM_IB_COM_RDMAWR_FROM]); #else conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = (uint64_t) buf_from; @@ -1298,16 +1373,22 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h int sz_data_red = sz_used + sz_data_rem + sizeof(MPID_nem_ib_tailmagic_t) <= MPID_NEM_IB_COM_INLINE_DATA ? sz_data_rem : sz_data_rem <= - MPID_NEM_IB_COM_INLINE_DATA - sz_used ? sz_data_rem : MPID_NEM_IB_COM_INLINE_DATA - sz_used; - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_data_red; + MPID_NEM_IB_COM_INLINE_DATA - sz_used ? sz_data_rem : MPID_NEM_IB_COM_INLINE_DATA - + sz_used; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = + sz_data_red; sz_used += sz_data_red; sz_data_rem -= sz_data_red; - MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem < 0, -1, printf("MPID_nem_ib_com_isend_chain,sz_data_rem\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem < 0, -1, + printf("MPID_nem_ib_com_isend_chain,sz_data_rem\n")); if (i == s) { - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, printf("MPID_nem_ib_com_isend_chain,sz_data==0\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, + printf("MPID_nem_ib_com_isend_chain,sz_data==0\n")); mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, + printf + ("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); } #ifdef HAVE_LIBDCFA conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = @@ -1315,9 +1396,11 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = mr_data->host_addr + ((uint64_t) data + sz_data - sz_data_rem - (uint64_t) data); #endif - conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = mr_data->lkey; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = + mr_data->lkey; num_sge += 1; - dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used, sz_data_rem); + dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used, + sz_data_rem); } else { /* tailmagic only packet is being generated */ @@ -1325,17 +1408,19 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h //tsce = MPID_nem_ib_rdtsc(); printf("1,%ld\n", tsce-tscs); //tscs = MPID_nem_ib_rdtsc(); - if (i == MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */ + if (i == MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */ int sz_pad = sz_data_pow2 - sz_data; - MPID_nem_ib_tailmagic_t *tailmagic = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sz_pad); + MPID_nem_ib_tailmagic_t *tailmagic = + (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sz_pad); tailmagic->magic = MPID_NEM_IB_COM_MAGIC; #ifdef HAVE_LIBDCFA conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = (uint64_t) buf_from; conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) conp-> - icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); + (uint64_t) + conp->icom_mem + [MPID_NEM_IB_COM_RDMAWR_FROM]); #else conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = (uint64_t) buf_from; @@ -1343,27 +1428,29 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad + sizeof(MPID_nem_ib_tailmagic_t); sz_used += sz_pad + sizeof(MPID_nem_ib_tailmagic_t); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem != 0, -1, printf("MPID_nem_ib_com_isend_chain, sz_data_rem\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem != 0, -1, + printf("MPID_nem_ib_com_isend_chain, sz_data_rem\n")); conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; num_sge += 1; conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].imm_data = conp->sseq_num; - dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad, sz_used, - num_sge); + dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad, + sz_used, num_sge); } - else if (MPID_NEM_IB_COM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */ + else if (MPID_NEM_IB_COM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */ MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, - printf - ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); + printf + ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); int sz_pad = MPID_NEM_IB_COM_INLINE_DATA - sz_used; #ifdef HAVE_LIBDCFA conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = (uint64_t) buf_from; conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) conp-> - icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); + (uint64_t) + conp->icom_mem + [MPID_NEM_IB_COM_RDMAWR_FROM]); #else conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = (uint64_t) buf_from; @@ -1372,7 +1459,7 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad; sz_used += sz_pad; MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1, - printf("MPID_nem_ib_com_isend_chain, sz_used\n")); + printf("MPID_nem_ib_com_isend_chain, sz_used\n")); conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; num_sge += 1; @@ -1380,7 +1467,7 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h } else { /* packet is full with data */ MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1, - printf("MPID_nem_ib_com_isend_chain, sz_used\n")); + printf("MPID_nem_ib_com_isend_chain, sz_used\n")); } //tsce = MPID_nem_ib_rdtsc(); printf("2,%ld\n", tsce-tscs); @@ -1388,8 +1475,8 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr_id = wr_id; conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.remote_addr = (uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] + - MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG) + MPID_NEM_IB_COM_INLINE_DATA * (i - - s); + MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG) + + MPID_NEM_IB_COM_INLINE_DATA * (i - s); } #if 0 if (conp->after_rdma_rd) { @@ -1397,9 +1484,12 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h } #endif #ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s]); + ib_errno = + ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s]); #else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s], &bad_wr); + ib_errno = + ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s], + &bad_wr); #endif #if 0 if (i == 0 && conp->after_rdma_rd) { @@ -1408,11 +1498,14 @@ int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_h } #endif #ifdef HAVE_LIBDCFA - MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", ib_errno)); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", + ib_errno)); #else MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); + dprintf + ("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); #endif conp->ncom += (MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - s); conp->sseq_num += 1; @@ -1439,14 +1532,15 @@ int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id) #ifdef HAVE_LIBDCFA ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER]); #else - ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER], &bad_wr); + ib_errno = + ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER], &bad_wr); #endif if (ib_errno) { #ifdef HAVE_LIBDCFA fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d\n", ib_errno); #else - fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n", ib_errno, - bad_wr); + fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n", + ib_errno, bad_wr); #endif ibcom_errno = ib_errno; goto fn_fail; @@ -1457,8 +1551,8 @@ int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id) goto fn_exit; } -int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, uint32_t remote_qpn, - uint32_t imm_data, uint64_t wr_id) +int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, + uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id) { MPID_nem_ib_com_t *conp; struct ibv_send_wr *bad_wr; @@ -1468,8 +1562,8 @@ int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remo #ifdef HAVE_LIBDCFA MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, - dprintf - ("MPID_nem_ib_com_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n")); + dprintf + ("MPID_nem_ib_com_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n")); #else /* prepare ibv_ah_attr */ conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].dlid = remote_lid; @@ -1542,7 +1636,8 @@ int MPID_nem_ib_com_udrecv(int condesc) goto fn_exit; } -int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr) +int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, + void *laddr) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1553,14 +1648,16 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, dprintf("MPID_nem_ib_com_lrecv,enter,raddr=%p,sz_data=%d,laddr=%p\n", raddr, sz_data, laddr); MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("MPID_nem_ib_com_lrecv,not connected\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, + dprintf("MPID_nem_ib_com_lrecv,not connected\n")); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("MPID_nem_ib_com_lrecv,sz_data==0\n")); num_sge = 0; /* register memory area containing data */ struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz_data); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_lrecv,ibv_reg_mr_fetch failed\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, + dprintf("MPID_nem_ib_com_lrecv,ibv_reg_mr_fetch failed\n")); /* Erase magic, super bug!! */ //((MPID_nem_ib_tailmagic_t*)(laddr + sz_data - sizeof(MPID_nem_ib_tailmagic_t)))->magic = 0; @@ -1582,12 +1679,15 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR]); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d\n", ib_errno)); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d\n", + ib_errno)); #else ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR], &bad_wr); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); + dprintf + ("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); #endif /* other commands can be executed before RDMA-rd command */ @@ -1604,7 +1704,8 @@ int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, } /* use the same QP as isend */ -int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr) +int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, + void *laddr) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1615,14 +1716,16 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat dprintf("MPID_nem_ib_com_put_lmt,enter,sz_data=%d,laddr=%p\n", sz_data, laddr); MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("MPID_nem_ib_com_put_lmt,not connected\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, + dprintf("MPID_nem_ib_com_put_lmt,not connected\n")); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("MPID_nem_ib_com_put_lmt,sz_data==0\n")); num_sge = 0; /* register memory area containing data */ struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz_data); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_lmt,ibv_reg_mr_fetch failed\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, + dprintf("MPID_nem_ib_com_put_lmt,ibv_reg_mr_fetch failed\n")); #ifdef HAVE_LIBDCFA conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].mic_addr = (uint64_t) laddr; @@ -1642,12 +1745,15 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT]); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d\n", ib_errno)); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d\n", + ib_errno)); #else ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT], &bad_wr); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); + dprintf + ("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); #endif conp->ncom += 1; @@ -1659,7 +1765,8 @@ int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_dat goto fn_exit; } -int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr) +int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, + void *laddr) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1672,14 +1779,17 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->open_flag != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag)); + dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", + conp->open_flag)); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n")); + dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n")); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,sz==0\n")); /* register memory area containing data */ struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, + dprintf + ("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n")); dprintf("MPID_nem_ib_com_put_scratch_pad,"); #ifdef HAVE_LIBDCFA @@ -1699,17 +1809,23 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset /* rkey is defined in MPID_nem_ib_com_reg_mr_connect */ dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr); + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma. + remote_addr); #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno)); + dprintf + ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", + ib_errno)); #else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], &bad_wr); + ib_errno = + ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], + &bad_wr); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", - ib_errno, bad_wr)); + dprintf + ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); #endif conp->ncom_scratch_pad += 1; @@ -1722,7 +1838,7 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset #ifdef MPID_NEM_IB_ONDEMAND int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, uint64_t compare, - uint64_t swap) + uint64_t swap) { MPID_nem_ib_com_t *conp; int ibcom_errno = 0; @@ -1735,14 +1851,17 @@ int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->open_flag != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag)); + dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", + conp->open_flag)); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n")); + dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n")); MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,sz==0\n")); /* register memory area containing data */ struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, + dprintf + ("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n")); dprintf("MPID_nem_ib_com_put_scratch_pad,"); #ifdef HAVE_LIBDCFA @@ -1764,17 +1883,23 @@ int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.atomic.swap = swap; dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr); + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma. + remote_addr); #ifdef HAVE_LIBDCFA ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno)); + dprintf + ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", + ib_errno)); #else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], &bad_wr); + ib_errno = + ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], + &bad_wr); MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", - ib_errno, bad_wr)); + dprintf + ("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); #endif conp->ncom_scratch_pad += 1; @@ -1807,8 +1932,9 @@ int MPID_nem_ib_com_poll_cq(int which_cq, struct ibv_wc *wc, int *result) } if (*result < 0) { - dprintf("MPID_nem_ib_com_poll_cq,status=%08x,vendor_err=%08x,len=%d,opcode=%08x,wr_id=%016lx\n", - wc->status, wc->vendor_err, wc->byte_len, wc->opcode, wc->wr_id); + dprintf + ("MPID_nem_ib_com_poll_cq,status=%08x,vendor_err=%08x,len=%d,opcode=%08x,wr_id=%016lx\n", + wc->status, wc->vendor_err, wc->byte_len, wc->opcode, wc->wr_id); ibcom_errno = *result; goto fn_fail; } @@ -1831,7 +1957,8 @@ int MPID_nem_ib_com_reg_mr_connect(int condesc, void *rmem, int rkey) case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] = rmem; conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO] = rkey; - conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey = conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO]; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey = + conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO]; for (i = 0; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.rkey = conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO]; @@ -1906,7 +2033,7 @@ int MPID_nem_ib_com_get_info_mr(int condesc, int memid, int key, void *out, int MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); MPID_NEM_IB_COM_ERR_CHKANDJUMP(memid >= conp->icom_mrlen, -1, - dprintf("MPID_nem_ib_com_get_info_mr,wrong mem_id=%d\n", memid)); + dprintf("MPID_nem_ib_com_get_info_mr,wrong mem_id=%d\n", memid)); mr = conp->icom_mrlist[memid]; switch (key) { @@ -1965,7 +2092,9 @@ int MPID_nem_ib_com_mem_rdmawr_to(int condesc, int seq_num, void **out) int ibcom_errno = 0; MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZSEG * (seq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); + *out = + (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * (seq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); fn_exit: return ibcom_errno; @@ -2179,7 +2308,8 @@ int MPID_nem_ib_com_reg_mr(void *addr, int len, struct ibv_mr **mr) ibv_reg_mr(ib_pd, addr, len, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ); - MPID_NEM_IB_COM_ERR_CHKANDJUMP(*mr == 0, -1, dprintf("MPID_nem_ib_com_reg_mr,cannot register memory\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(*mr == 0, -1, + dprintf("MPID_nem_ib_com_reg_mr,cannot register memory\n")); fn_exit: return ibcom_errno; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h index e77091e..930fbf1 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h @@ -175,15 +175,15 @@ extern struct ibv_cq *MPID_nem_ib_rc_shared_scq_scratch_pad; extern struct ibv_cq *MPID_nem_ib_ud_shared_rcq; #define MPID_NEM_IB_COM_SIZE 2048 /* one process uses 2-4 fds */ -#define MPID_NEM_IB_COM_INLINE_DATA (512-64) /* experimented max is 884 */ /* this is lower bound and more than this value is set. the more this value is, the more the actual value set is. you need to check it */ +#define MPID_NEM_IB_COM_INLINE_DATA (512-64) /* experimented max is 884 */ /* this is lower bound and more than this value is set. the more this value is, the more the actual value set is. you need to check it */ #define MPID_NEM_IB_COM_MAX_SQ_CAPACITY (256/1) -#define MPID_NEM_IB_COM_MAX_RQ_CAPACITY ((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)+16) /* We pre-post_recv MPID_NEM_IB_COM_MAX_SQ_CAPACITY of commands */ -#define MPID_NEM_IB_COM_MAX_SGE_CAPACITY (32/2) /* maximum for ConnectX-3 looks like 32 */ +#define MPID_NEM_IB_COM_MAX_RQ_CAPACITY ((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)+16) /* We pre-post_recv MPID_NEM_IB_COM_MAX_SQ_CAPACITY of commands */ +#define MPID_NEM_IB_COM_MAX_SGE_CAPACITY (32/2) /* maximum for ConnectX-3 looks like 32 */ #define MPID_NEM_IB_COM_MAX_CQ_CAPACITY MPID_NEM_IB_COM_MAX_RQ_CAPACITY -#define MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>1)) /* drain when reaching this amount */ -#define MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>1)) /* drain when reaching this amount */ -#define MPID_NEM_IB_COM_AMT_CQ_DRAIN ((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2) /* drain this amount */ +#define MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>1)) /* drain when reaching this amount */ +#define MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>1)) /* drain when reaching this amount */ +#define MPID_NEM_IB_COM_AMT_CQ_DRAIN ((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2) /* drain this amount */ #define MPID_NEM_IB_COM_MAX_RD_ATOMIC 4 #define MPID_NEM_IB_COM_MAX_TRIES 1 @@ -200,65 +200,65 @@ extern struct ibv_cq *MPID_nem_ib_ud_shared_rcq; /* buffers */ -#define MPID_NEM_IB_COM_NBUF_RDMA 2 /* number of <addr, sz, lkey, rkey> */ -#define MPID_NEM_IB_COM_RDMAWR_FROM 0 /* index to RDMA-write-from buffer */ -#define MPID_NEM_IB_COM_RDMAWR_TO 1 /* index to RDMA-write-to buffer */ +#define MPID_NEM_IB_COM_NBUF_RDMA 2 /* number of <addr, sz, lkey, rkey> */ +#define MPID_NEM_IB_COM_RDMAWR_FROM 0 /* index to RDMA-write-from buffer */ +#define MPID_NEM_IB_COM_RDMAWR_TO 1 /* index to RDMA-write-to buffer */ /* assuming that the unit (32768) is equals to eager-RDMA-write threashold assuming that the multiplier (256) is equals to max number of outstanding eager-RDMA-write transactions */ -#define MPID_NEM_IB_COM_RDMABUF_SZSEG (16384/4) //(16384+8+40+1) /* this size minus magics and headers must be 2^n because data might grow to the next 2^m boundary, see ib_impl.h, ib_com.c, src/mpid/ch3/src/mpid_isend.c */ -#define MPID_NEM_IB_COM_RDMABUF_SZ ((MPID_NEM_IB_COM_RDMABUF_SZSEG) * 16) /* (32768 * 256) */ +#define MPID_NEM_IB_COM_RDMABUF_SZSEG (16384/4) //(16384+8+40+1) /* this size minus magics and headers must be 2^n because data might grow to the next 2^m boundary, see ib_impl.h, ib_com.c, src/mpid/ch3/src/mpid_isend.c */ +#define MPID_NEM_IB_COM_RDMABUF_SZ ((MPID_NEM_IB_COM_RDMABUF_SZSEG) * 16) /* (32768 * 256) */ #define MPID_NEM_IB_COM_RDMABUF_NSEG ((MPID_NEM_IB_COM_RDMABUF_SZ) / (MPID_NEM_IB_COM_RDMABUF_SZSEG)) -#define MPID_NEM_IB_COM_SMT_INLINE_NCHAIN 8 /* maximum number of chained inline-send commands */ +#define MPID_NEM_IB_COM_SMT_INLINE_NCHAIN 8 /* maximum number of chained inline-send commands */ #define MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>1)+((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) #define MPID_NEM_IB_COM_RDMABUF_LOW_WATER_MARK (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) #define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW 1 #define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW 2 #define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW /*1*/(((MPID_NEM_IB_COM_RDMABUF_NSEG)>>4) == 0 ? 1 : ((MPID_NEM_IB_COM_RDMABUF_NSEG)>>4)) -#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */ -#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */ +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */ +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */ -#define MPID_NEM_IB_COM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */ -#define MPID_NEM_IB_COM_UDWR_FROM 0 /* index to UD-write-from buffer */ -#define MPID_NEM_IB_COM_UDWR_TO 1 /* index to UD-write-to buffer */ -#define MPID_NEM_IB_COM_UDBUF_SZ (128 * 8192) /* supporting 100K ranks with 10 rounds */ +#define MPID_NEM_IB_COM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */ +#define MPID_NEM_IB_COM_UDWR_FROM 0 /* index to UD-write-from buffer */ +#define MPID_NEM_IB_COM_UDWR_TO 1 /* index to UD-write-to buffer */ +#define MPID_NEM_IB_COM_UDBUF_SZ (128 * 8192) /* supporting 100K ranks with 10 rounds */ #define MPID_NEM_IB_COM_UDBUF_SZSEG (128) #define MPID_NEM_IB_COM_UDBUF_NSEG (MPID_NEM_IB_COM_UDBUF_SZ / MPID_NEM_IB_COM_UDBUF_SZSEG) -#define MPID_NEM_IB_COM_NBUF_SCRATCH_PAD 1 /* number of <addr, sz, lkey, rkey> */ -#define MPID_NEM_IB_COM_SCRATCH_PAD_TO 0 /* index to RDMA-write-to buffer */ +#define MPID_NEM_IB_COM_NBUF_SCRATCH_PAD 1 /* number of <addr, sz, lkey, rkey> */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_TO 0 /* index to RDMA-write-to buffer */ /* send command templates */ -#define MPID_NEM_IB_COM_RC_SR_NTEMPLATE (8+1+2) /* number of request templates, 8 for inline-chained-smt, 1 for smt, 1 for lmt */ -#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 0 /* index to it */ +#define MPID_NEM_IB_COM_RC_SR_NTEMPLATE (8+1+2) /* number of request templates, 8 for inline-chained-smt, 1 for smt, 1 for lmt */ +#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 0 /* index to it */ #define MPID_NEM_IB_COM_SMT_INLINE_CHAINED7 7 #define MPID_NEM_IB_COM_SMT_NOINLINE 8 -#define MPID_NEM_IB_COM_LMT_INITIATOR 9 /* FIXME: bad naming */ +#define MPID_NEM_IB_COM_LMT_INITIATOR 9 /* FIXME: bad naming */ -#define MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE MPID_NEM_IB_COM_RC_SR_NTEMPLATE /* FIXME: TEMPLATE named MPID_NEM_IB_COM_RC_SR shares MPID_NEM_IB_COM_LMT_PUT */ +#define MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE MPID_NEM_IB_COM_RC_SR_NTEMPLATE /* FIXME: TEMPLATE named MPID_NEM_IB_COM_RC_SR shares MPID_NEM_IB_COM_LMT_PUT */ #define MPID_NEM_IB_COM_LMT_PUT 10 /* recv command templates */ -#define MPID_NEM_IB_COM_RC_RR_NTEMPLATE 1 /* 1 for smt, */ -#define MPID_NEM_IB_COM_RDMAWR_RESPONDER 0 /* index to recv request template */ +#define MPID_NEM_IB_COM_RC_RR_NTEMPLATE 1 /* 1 for smt, */ +#define MPID_NEM_IB_COM_RDMAWR_RESPONDER 0 /* index to recv request template */ /* sge template */ -#define MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ -#define MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ -#define MPID_NEM_IB_COM_LMT_INITIATOR_NSGE 1 /* data x1 */ -#define MPID_NEM_IB_COM_LMT_PUT_NSGE 1 /* data x1 */ -#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE 1 /* QP state */ +#define MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ +#define MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ +#define MPID_NEM_IB_COM_LMT_INITIATOR_NSGE 1 /* data x1 */ +#define MPID_NEM_IB_COM_LMT_PUT_NSGE 1 /* data x1 */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE 1 /* QP state */ #define MPID_NEM_IB_COM_UD_SR_NTEMPLATE 1 #define MPID_NEM_IB_COM_UD_RR_NTEMPLATE 1 -#define MPID_NEM_IB_COM_UD_INITIATOR 0 /* index to send request template */ -#define MPID_NEM_IB_COM_UD_RESPONDER 0 /* index to recv request template */ +#define MPID_NEM_IB_COM_UD_INITIATOR 0 /* index to send request template */ +#define MPID_NEM_IB_COM_UD_RESPONDER 0 /* index to recv request template */ #define MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE 2 #define MPID_NEM_IB_COM_SCRATCH_PAD_RR_NTEMPLATE 1 -#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR 0 /* index to send request template */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR 0 /* index to send request template */ #define MPID_NEM_IB_COM_SCRATCH_PAD_CAS 1 -#define MPID_NEM_IB_COM_SCRATCH_PAD_RESPONDER 0 /* index to recv request template */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_RESPONDER 0 /* index to recv request template */ typedef struct MPID_nem_ib_com { @@ -310,23 +310,25 @@ typedef struct MPID_nem_ib_com { extern int MPID_nem_ib_com_open(int ib_port, int MPID_nem_ib_com_open_flag, int *condesc); extern int MPID_nem_ib_com_alloc(int condesc, int sz); extern int MPID_nem_ib_com_close(int); -extern int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid); +extern int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, + union ibv_gid *remote_gid); extern int MPID_nem_ib_com_reg_mr_connect(int condesc, void *rmem, int rkey); -extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, - int sz_hdr, void *data, int sz_data, int *copied); -extern int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, - int sz_data); -extern int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr); +extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, + void *hdr, int sz_hdr, void *data, int sz_data, int *copied); +extern int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, + void *data, int sz_data); +extern int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, + void *laddr); //extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void* hdr, int sz_hdr, void* data, int sz_data); extern int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id); extern int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, - uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id); + uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id); extern int MPID_nem_ib_com_udrecv(int condesc); -extern int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, - void *laddr); -extern int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, - void *laddr); +extern int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, + uint32_t rkey, void *laddr); +extern int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, + uint32_t rkey, void *laddr); extern int MPID_nem_ib_com_poll_cq(int which_cq, struct ibv_wc *wc, int *result); extern int MPID_nem_ib_com_obtain_pointer(int condesc, MPID_nem_ib_com_t ** MPID_nem_ib_com); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index aaddb5c..e8d230d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -15,7 +15,7 @@ #include <linux/mman.h> /* make it define MAP_ANONYMOUS */ #include <sys/mman.h> -#define MPID_NEM_IB_LMT_GET_CQE /* detect RDMA completion by CQE */ +#define MPID_NEM_IB_LMT_GET_CQE /* detect RDMA completion by CQE */ #define MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* lmt-put: (1) receiver sends cts to sender (2) sender RDMA-write to receiver @@ -44,7 +44,7 @@ typedef struct { MPID_nem_ib_conn_t *sc; int pending_sends; /* number of send in flight */ MPID_nem_ib_com_t *ibcom, *ibcom_lmt_put; - MPID_nem_ib_sendq_t sendq; /* overflow queue for IB commands */ + MPID_nem_ib_sendq_t sendq; /* overflow queue for IB commands */ MPID_nem_ib_sendq_t sendq_lmt_put; int is_connected; /* dynamic connection, checked in iSendContig, protocol processed there and in progress engine */ } MPID_nem_ib_vc_area; @@ -112,7 +112,7 @@ typedef union { typedef struct MPID_nem_ib_cm_sendq_entry { MPID_nem_ib_cm_pkt_t pending_pkt; - struct MPID_nem_ib_cm_sendq_entry *sendq_next; /* for software command queue */ + struct MPID_nem_ib_cm_sendq_entry *sendq_next; /* for software command queue */ } MPID_nem_ib_cm_sendq_entry_t; #ifdef MPID_NEM_IB_ONDEMAND @@ -157,10 +157,10 @@ typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_ib_cm_sendq_t; /* counting bloom filter to detect multiple lmt-sends in one send-wait period to avoid overwriting the last byte in the receive buffer */ -#define MPID_nem_ib_cbf_nslot 16 /* slots */ -#define MPID_nem_ib_cbf_bitsperslot 4 /* one slot can accomodate multiple bits */ +#define MPID_nem_ib_cbf_nslot 16 /* slots */ +#define MPID_nem_ib_cbf_bitsperslot 4 /* one slot can accomodate multiple bits */ #define MPID_nem_ib_cbf_lognslot 4 -#define MPID_nem_ib_cbf_nhash 3 /* number of hash functions */ +#define MPID_nem_ib_cbf_nhash 3 /* number of hash functions */ #define MPID_nem_ib_getpos \ int pos_8b = pos / (8 / MPID_nem_ib_cbf_bitsperslot);\ assert(0 <= pos_8b && pos_8b < MPID_nem_ib_cbf_nslot * MPID_nem_ib_cbf_bitsperslot / 8);\ @@ -266,13 +266,13 @@ static inline int MPID_nem_ib_cbf_would_overflow(uint64_t addr, uint8_t * array) //dprintf("cbf_would_overflow,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); return MPID_nem_ib_cbf_get(array, - MPID_nem_ib_cbf_hash1(addr)) == + MPID_nem_ib_cbf_hash1(addr)) == (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1 || MPID_nem_ib_cbf_get(array, - MPID_nem_ib_cbf_hash2(addr)) == + MPID_nem_ib_cbf_hash2(addr)) == (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1 || MPID_nem_ib_cbf_get(array, - MPID_nem_ib_cbf_hash3(addr)) == + MPID_nem_ib_cbf_hash3(addr)) == (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1; } @@ -295,22 +295,22 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc); int MPID_nem_ib_pkthandler_init(MPIDI_CH3_PktHandler_Fcn * pktArray[], int arraySize); int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header, - MPIDI_msg_sz_t hdr_sz); + MPIDI_msg_sz_t hdr_sz); /* CH3 send/recv functions */ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, - MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz); + MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz); int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, - MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr); + MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr); /* used by ib_poll.c */ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib); /* CH3--lmt send/recv functions */ int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, - struct MPID_Request *req); + struct MPID_Request *req); int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, - void *write_to_buf); + void *write_to_buf); int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie); int MPID_nem_ib_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie); int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req); @@ -345,24 +345,24 @@ extern void *MPID_nem_ib_fl[18]; extern int MPID_nem_ib_nranks; //extern char *MPID_nem_ib_recv_buf; extern int MPID_nem_ib_myrank; -extern uint64_t MPID_nem_ib_tsc_poll; /* to throttle ib_poll in recv_posted (in ib_poll.c) */ -extern int MPID_nem_ib_ncqe; /* for lazy poll scq */ -extern int MPID_nem_ib_ncqe_lmt_put; /* lmt-put uses another QP, SQ, CQ to speed-up fetching CQE */ +extern uint64_t MPID_nem_ib_tsc_poll; /* to throttle ib_poll in recv_posted (in ib_poll.c) */ +extern int MPID_nem_ib_ncqe; /* for lazy poll scq */ +extern int MPID_nem_ib_ncqe_lmt_put; /* lmt-put uses another QP, SQ, CQ to speed-up fetching CQE */ #ifdef MPID_NEM_IB_ONDEMAND extern MPID_nem_ib_cm_map_t MPID_nem_ib_cm_state; -extern int MPID_nem_ib_ncqe_connect; /* couting outstanding connection requests */ +extern int MPID_nem_ib_ncqe_connect; /* couting outstanding connection requests */ #endif extern int MPID_nem_ib_ncqe_scratch_pad; -extern int MPID_nem_ib_ncqe_to_drain; /* count put in lmt-put-done protocol */ -extern int MPID_nem_ib_ncqe_nces; /* counting non-copied eager-send */ -extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq; /* poll queue for lmt */ -extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq; /* force order when two or more rts_to_sender randomizes the last byte of receive buffer */ +extern int MPID_nem_ib_ncqe_to_drain; /* count put in lmt-put-done protocol */ +extern int MPID_nem_ib_ncqe_nces; /* counting non-copied eager-send */ +extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq; /* poll queue for lmt */ +extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq; /* force order when two or more rts_to_sender randomizes the last byte of receive buffer */ extern MPID_nem_ib_vc_area *MPID_nem_ib_debug_current_vc_ib; /* to detect multiple lmt-sends in one send-wait period to avoid overwriting the last byte in the receive buffer */ extern uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * - MPID_nem_ib_cbf_bitsperslot / 8]; + MPID_nem_ib_cbf_bitsperslot / 8]; #define MPID_NEM_IB_MAX_POLLINGSET 65536 @@ -374,8 +374,8 @@ extern uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * #define MPID_NEM_IB_SYNC_SYNACK 1 #define MPID_NEM_IB_SYNC_NACK 2 -#define MPID_NEM_IB_EAGER_MAX_MSG_SZ (MPID_NEM_IB_COM_RDMABUF_SZSEG/*1024*/-sizeof(MPIDI_CH3_Pkt_t)+sizeof(MPIDI_CH3_Pkt_eager_send_t)-sizeof(MPID_nem_ib_sz_hdrmagic_t)-sizeof(MPID_nem_ib_pkt_prefix_t)-sizeof(MPID_nem_ib_tailmagic_t)) /* when > this size, lmt is used. see src/mpid/ch3/src/mpid_isend.c */ -#define MPID_NEM_IB_POLL_PERIOD_RECV_POSTED 2000 /* minimum period from previous ib_poll to ib_poll in recv_posted */ +#define MPID_NEM_IB_EAGER_MAX_MSG_SZ (MPID_NEM_IB_COM_RDMABUF_SZSEG/*1024*/-sizeof(MPIDI_CH3_Pkt_t)+sizeof(MPIDI_CH3_Pkt_eager_send_t)-sizeof(MPID_nem_ib_sz_hdrmagic_t)-sizeof(MPID_nem_ib_pkt_prefix_t)-sizeof(MPID_nem_ib_tailmagic_t)) /* when > this size, lmt is used. see src/mpid/ch3/src/mpid_isend.c */ +#define MPID_NEM_IB_POLL_PERIOD_RECV_POSTED 2000 /* minimum period from previous ib_poll to ib_poll in recv_posted */ #define MPID_NEM_IB_POLL_PERIOD_SEND_POSTED 2000 typedef struct { @@ -440,32 +440,32 @@ typedef struct MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t { } MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t; int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, - MPID_Request ** rreqp); + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, + MPID_Request ** rreqp); /* MPID_nem_ib_PktHandler_lmt_done is a wrapper of pkt_DONE_handler and calls it */ /* pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) is not exported */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c index 25cd464..1b969d8 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c @@ -40,8 +40,8 @@ MPID_nem_netmod_funcs_t MPIDI_nem_ib_funcs = { MPIDI_CH3_PktHandler_Fcn *MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_NUM_PKT_HANDLERS]; static MPIDI_Comm_ops_t comm_ops = { - /*NULL, */ MPID_nem_ib_recv_posted, - /* recv_posted */ + /*NULL, */ MPID_nem_ib_recv_posted, + /* recv_posted */ NULL, /* send */ NULL, /* rsend */ @@ -88,7 +88,7 @@ int MPID_nem_ib_ncqe_nces; MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq = { NULL, NULL }; MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq = { NULL, NULL }; uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * MPID_nem_ib_cbf_bitsperslot / - 8] = { 0 }; + 8] = { 0 }; static uint32_t MPID_nem_ib_rand_next = 1; MPID_nem_ib_vc_area *MPID_nem_ib_debug_current_vc_ib; static int listen_fd; @@ -112,7 +112,8 @@ uint64_t MPID_nem_ib_rdtsc() #define FUNCNAME MPID_nem_ib_kvs_put_binary #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPID_nem_ib_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, int length) +static int MPID_nem_ib_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, + int length) { int mpi_errno = MPI_SUCCESS; int pmi_errno; @@ -223,7 +224,9 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m /* prepare UD QPN for dynamic connection */ ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_UD, &MPID_nem_ib_conn_ud_fd); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); - ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conn_ud_fd, &MPID_nem_ib_conn_ud_MPID_nem_ib_com); + ibcom_errno = + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conn_ud_fd, + &MPID_nem_ib_conn_ud_MPID_nem_ib_com); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); ibcom_errno = MPID_nem_ib_com_rts(MPID_nem_ib_conn_ud_fd, 0, 0, 0); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); @@ -244,11 +247,11 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m uint16_t my_lid; union ibv_gid my_gid; MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); + sizeof(uint32_t)); MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid, - sizeof(uint16_t)); + sizeof(uint16_t)); MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid, - sizeof(union ibv_gid)); + sizeof(union ibv_gid)); char *kvs_name; mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); @@ -295,15 +298,13 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m MPID_nem_ib_conn_ud[i].gid.raw[j] = strtol(str, NULL, 16); strp += 2; } - sscanf(strp, ":%04x:%08x", &MPID_nem_ib_conn_ud[i].lid, - &MPID_nem_ib_conn_ud[i].qpn); + sscanf(strp, ":%04x:%08x", &MPID_nem_ib_conn_ud[i].lid, &MPID_nem_ib_conn_ud[i].qpn); dprintf("remote rank=%d,gid=", i); for (j = 0; j < sizeof(union ibv_gid); j++) { dprintf("%02x", MPID_nem_ib_conn_ud[i].gid.raw[j]); } - dprintf(",lid=%04x,qpn=%08x\n", MPID_nem_ib_conn_ud[i].lid, - MPID_nem_ib_conn_ud[i].qpn); + dprintf(",lid=%04x,qpn=%08x\n", MPID_nem_ib_conn_ud[i].lid, MPID_nem_ib_conn_ud[i].qpn); } } #endif @@ -315,12 +316,14 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m /* prepare scrath-pad QP and malloc scratch-pad */ for (i = 0; i < MPID_nem_ib_nranks; i++) { - ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, &MPID_nem_ib_scratch_pad_fds[i]); + ibcom_errno = + MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, + &MPID_nem_ib_scratch_pad_fds[i]); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); ibcom_errno = MPID_nem_ib_com_alloc(MPID_nem_ib_scratch_pad_fds[i], - MPID_nem_ib_nranks * sizeof(MPID_nem_ib_com_qp_state_t)); + MPID_nem_ib_nranks * sizeof(MPID_nem_ib_com_qp_state_t)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_alloc"); } @@ -355,14 +358,16 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m if (i == 0) { ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_PORT_LID, - &my_lid, sizeof(uint16_t)); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], + MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid, + sizeof(uint16_t)); dprintf("ib_init,scratch pad,lid=%04x\n", my_lid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, "sp/lid", (uint8_t *) & my_lid, - sizeof(uint16_t)); + sizeof(uint16_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); @@ -372,13 +377,15 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m } ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_PORT_GID, - &my_gid, sizeof(union ibv_gid)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], + MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid, + sizeof(union ibv_gid)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, "sp/gid", (uint8_t *) & my_gid, - sizeof(union ibv_gid)); + sizeof(union ibv_gid)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); @@ -394,14 +401,16 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m sprintf(remote_rank_str, "/%x", i); strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], + MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); dprintf("ib_init,scratch pad,qpn=%08x\n", my_qpnum); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum, - sizeof(uint32_t)); + sizeof(uint32_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); dprintf("ib_init,scratch pad,kvs put done\n"); @@ -410,14 +419,15 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_SCRATCH_PAD_TO, - MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *)); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], + MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); dprintf("ib_init,scratch_pad,rmem=%p\n", my_rmem); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rmem, - sizeof(void *)); + sizeof(void *)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); strcpy(key_str, "sp/rkey"); @@ -425,14 +435,15 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_SCRATCH_PAD_TO, - MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int)); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], + MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); dprintf("ib_init,scratch_pad,rkey=%08x\n", my_rkey); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rkey, - sizeof(uint32_t)); + sizeof(uint32_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); } @@ -447,7 +458,7 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m mpi_errno = MPID_nem_ib_kvs_get_binary(i, "sp/gid", (char *) &remote_gid, - sizeof(union ibv_gid)); + sizeof(union ibv_gid)); dprintf("ib_init,after kvs get\n"); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -472,7 +483,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m dprintf("ib_init,get KVS,remote_qpnum=%08x\n", remote_qpnum); ibcom_errno = - MPID_nem_ib_com_rts(MPID_nem_ib_scratch_pad_fds[i], remote_qpnum, remote_lid, &remote_gid); + MPID_nem_ib_com_rts(MPID_nem_ib_scratch_pad_fds[i], remote_qpnum, remote_lid, + &remote_gid); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); strcpy(key_str, "sp/rmem"); @@ -497,8 +509,10 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m dprintf("ib_init,get KVS,remote_rkey=%08x\n", remote_rkey); ibcom_errno = - MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_scratch_pad_fds[i], remote_rmem, remote_rkey); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect"); + MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_scratch_pad_fds[i], remote_rmem, + remote_rkey); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_reg_mr_connect"); } } @@ -507,7 +521,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); MPIU_CHKPMEM_MALLOC(MPID_nem_ib_conns, MPID_nem_ib_conn_t *, - MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t), mpi_errno, "connection table"); + MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t), mpi_errno, + "connection table"); memset(MPID_nem_ib_conns, 0, MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t)); MPIU_CHKPMEM_MALLOC(MPID_nem_ib_pollingset, MPIDI_VC_t **, @@ -517,14 +532,17 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m /* prepare eager-send QP */ for (i = 0; i < MPID_nem_ib_nranks; i++) { - ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[i].fd); + ibcom_errno = + MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[i].fd); dprintf("init,fd=%d\n", MPID_nem_ib_conns[i].fd); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); } #if 0 for (i = 0; i < MPID_nem_ib_nranks; i++) { - ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC_LMT_PUT, &MPID_nem_ib_conns[i].fd_lmt_put); + ibcom_errno = + MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC_LMT_PUT, + &MPID_nem_ib_conns[i].fd_lmt_put); dprintf("init,fd_lmt_put=%d\n", MPID_nem_ib_conns[i].fd_lmt_put); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); } @@ -546,13 +564,13 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m mpi_errno = MPID_nem_ib_kvs_get_binary(i, MPID_NEM_IB_LID_KEY, (char *) &remote_lid, - sizeof(uint16_t)); + sizeof(uint16_t)); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } mpi_errno = MPID_nem_ib_kvs_get_binary(i, MPID_NEM_IB_GID_KEY, (char *) &remote_gid, - sizeof(union ibv_gid)); + sizeof(union ibv_gid)); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -593,20 +611,27 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m /* report me-to-you eager-send QP becomes RTR */ MPID_nem_ib_com_t *MPID_nem_ib_com_scratch_pad; ibcom_errno = - MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[i], &MPID_nem_ib_com_scratch_pad); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[i], + &MPID_nem_ib_com_scratch_pad); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_obtain_pointer"); MPID_nem_ib_com_qp_state_t state = {.state = MPID_NEM_IB_COM_QP_STATE_RTR }; ibcom_errno = MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[i], - (uint64_t) MPID_nem_ib_com_scratch_pad, - sizeof(MPID_nem_ib_com_qp_state_t) * MPID_nem_ib_myrank, - sizeof(MPID_nem_ib_com_qp_state_t), (void *) &state); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_put_scratch_pad"); + (uint64_t) MPID_nem_ib_com_scratch_pad, + sizeof(MPID_nem_ib_com_qp_state_t) * + MPID_nem_ib_myrank, + sizeof(MPID_nem_ib_com_qp_state_t), + (void *) &state); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_put_scratch_pad"); MPID_nem_ib_ncqe_scratch_pad += 1; - ibcom_errno = MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[i].fd, remote_rmem, remote_rkey); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect"); + ibcom_errno = + MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[i].fd, remote_rmem, remote_rkey); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_reg_mr_connect"); dprintf("ib_init,after mr_connect for me-to-you eager-send QP\n"); #if 0 @@ -622,7 +647,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m } ibcom_errno = - MPID_nem_ib_com_rts(MPID_nem_ib_conns[i].fd_lmt_put, remote_qpnum, remote_lid, &remote_gid); + MPID_nem_ib_com_rts(MPID_nem_ib_conns[i].fd_lmt_put, remote_qpnum, remote_lid, + &remote_gid); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); #endif } @@ -685,7 +711,7 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * for (i = 0, nranks = MPID_nem_ib_nranks; nranks > 0; nranks /= 10, i++) { } MPIU_CHKLMEM_MALLOC(remote_rank_str, char *, i + 1, mpi_errno, "key_str"); /* alloca */ - MPIU_CHKLMEM_MALLOC(key_str, char *, strlen(MPID_NEM_IB_QPN_KEY) + i + 1, mpi_errno, "key_str"); /* alloca */ + MPIU_CHKLMEM_MALLOC(key_str, char *, strlen(MPID_NEM_IB_QPN_KEY) + i + 1, mpi_errno, "key_str"); /* alloca */ /* We have one local qp and remote qp for each rank-pair, * so a rank should perform @@ -700,20 +726,23 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * /* lid and gid are common for all remote-ranks */ if (i == 0) { ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid, - sizeof(uint16_t)); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, + MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid, + sizeof(uint16_t)); dprintf("get_business_card,lid=%04x\n", my_lid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, MPID_NEM_IB_LID_KEY, - (uint8_t *) & my_lid, sizeof(uint16_t)); + (uint8_t *) & my_lid, sizeof(uint16_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid, - sizeof(union ibv_gid)); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, + MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid, + sizeof(union ibv_gid)); dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); dprintf("get_business_card,sz=%ld,my_gid=", sizeof(union ibv_gid)); @@ -722,11 +751,12 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * } dprintf("\n"); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, MPID_NEM_IB_GID_KEY, - (uint8_t *) & my_gid, sizeof(union ibv_gid)); + (uint8_t *) & my_gid, sizeof(union ibv_gid)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); @@ -743,13 +773,14 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * sprintf(remote_rank_str, "/%x", i); strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, + &my_qpnum, sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum, - sizeof(uint32_t)); + sizeof(uint32_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); #if 0 @@ -759,13 +790,15 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * sprintf(remote_rank_str, "/%x", i); strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd_lmt_put, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd_lmt_put, + MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum, - sizeof(uint32_t)); + sizeof(uint32_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); #endif @@ -774,14 +807,14 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, MPID_NEM_IB_COM_INFOKEY_MR_ADDR, - &my_rmem, sizeof(void *)); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); dprintf("rmem=%p\n", my_rmem); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rmem, - sizeof(void *)); + sizeof(void *)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); strcpy(key_str, MPID_NEM_IB_RKEY_KEY); @@ -789,13 +822,13 @@ static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int * strcat(key_str, remote_rank_str); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, MPID_NEM_IB_COM_INFOKEY_MR_RKEY, - &my_rkey, sizeof(int)); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); mpi_errno = MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rkey, - sizeof(uint32_t)); + sizeof(uint32_t)); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); } @@ -857,7 +890,8 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); ibcom_errno = - MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd_lmt_put, &vc_ib->ibcom_lmt_put); + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd_lmt_put, + &vc_ib->ibcom_lmt_put); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); //dprintf("vc_init,open,fd=%d,ptr=%p,rsr_seq_num_poll=%d\n", MPID_nem_ib_conns[vc->pg_rank].fd, vc_ib->MPID_nem_ib_com, vc_ib->ibcom->rsr_seq_num_poll); @@ -878,13 +912,15 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) /* wait until you-to-me eager-send QP becomes RTR */ MPID_nem_ib_com_t *MPID_nem_ib_com_scratch_pad; ibcom_errno = - MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[vc->pg_rank], &MPID_nem_ib_com_scratch_pad); + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[vc->pg_rank], + &MPID_nem_ib_com_scratch_pad); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); int ntrial = 0; volatile MPID_nem_ib_com_qp_state_t *rstate = - (MPID_nem_ib_com_qp_state_t *) ((uint8_t *) MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + - vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t)); + (MPID_nem_ib_com_qp_state_t *) ((uint8_t *) MPID_nem_ib_com_scratch_pad-> + icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + + vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t)); dprintf("ib_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate)); while (rstate->state != MPID_NEM_IB_COM_QP_STATE_RTR) { __asm__ __volatile__("pause;":::"memory"); @@ -901,7 +937,8 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) int i; for (i = 0; i < MPID_NEM_IB_COM_MAX_RQ_CAPACITY; i++) { //dprintf("irecv,%d->%d\n", MPID_nem_ib_myrank, vc->pg_rank); - ibcom_errno = MPID_nem_ib_com_irecv(MPID_nem_ib_conns[vc->pg_rank].fd, (uint64_t) vc->pg_rank); + ibcom_errno = + MPID_nem_ib_com_irecv(MPID_nem_ib_conns[vc->pg_rank].fd, (uint64_t) vc->pg_rank); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_irecv"); } @@ -909,8 +946,9 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) uint32_t max_msg_sz; - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[vc->pg_rank].fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, - &max_msg_sz, sizeof(max_msg_sz)); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[vc->pg_rank].fd, + MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, &max_msg_sz, + sizeof(max_msg_sz)); VC_FIELD(vc, pending_sends) = 0; #ifdef MPID_NEM_IB_ONDEMAND VC_FIELD(vc, is_connected) = 0; @@ -946,7 +984,7 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) dprintf("ib_vc_init,vc->eager_max_msg_sz=%d\n", vc->eager_max_msg_sz); /* vc->rndvSend_fn is set in MPID_nem_vc_init (in src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c) */ - ; + ; vc->sendNoncontig_fn = MPID_nem_ib_SendNoncontig; vc->comm_ops = &comm_ops; @@ -962,8 +1000,7 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_ACCUMULATE] = MPID_nem_ib_PktHandler_Accumulate; MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_LMT_GET_DONE] = MPID_nem_ib_pkt_GET_DONE_handler; MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REQ_SEQ_NUM] = MPID_nem_ib_PktHandler_req_seq_num; - MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM] = - MPID_nem_ib_PktHandler_reply_seq_num; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM] = MPID_nem_ib_PktHandler_reply_seq_num; MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_CHG_RDMABUF_OCC_NOTIFY_STATE] = MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state; @@ -1025,10 +1062,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) dprintf("init,before,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail_last_sent), MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), - MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, - pending_sends)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); /* update remote RDMA-write-to buffer occupancy */ #if 0 /* we can't send it when the other party has closed QP */ @@ -1053,10 +1089,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) dprintf("init,middle,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail_last_sent), MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), - MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, - pending_sends)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { usleep(1000); @@ -1065,10 +1100,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) dprintf("init,middle2,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail_last_sent), MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), - MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, - pending_sends)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { usleep(1000); @@ -1087,10 +1121,9 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) dprintf("init,after ,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail_last_sent), MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), - MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, - pending_sends)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends)); /* drain scratch-pad scq */ ibcom_errno = MPID_nem_ib_drain_scq_scratch_pad(); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index f0c6ea4..e5da2f9 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -23,7 +23,7 @@ #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, - struct MPID_Request *req) + struct MPID_Request *req) { int mpi_errno = MPI_SUCCESS; int dt_contig; @@ -137,7 +137,7 @@ int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_p #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, - void *write_to_buf) + void *write_to_buf) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -149,7 +149,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3 ibcom_errno = MPID_nem_ib_com_lrecv(vc_ib->sc->fd, (uint64_t) req, raddr, req->ch.lmt_data_sz, rkey, - write_to_buf); + write_to_buf); MPID_nem_ib_ncqe += 1; //dprintf("start_recv,ncqe=%d\n", MPID_nem_ib_ncqe); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lrecv"); @@ -161,7 +161,7 @@ int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint3 *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t)))); #ifdef MPID_NEM_IB_LMT_GET_CQE - MPID_nem_ib_ncqe_to_drain += 1; /* use CQE instead of polling */ + MPID_nem_ib_ncqe_to_drain += 1; /* use CQE instead of polling */ #else /* drain_scq and ib_poll is not ordered, so both can decrement ref_count */ MPIR_Request_add_ref(req); @@ -250,7 +250,7 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { - mpi_errno = MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, write_to_buf); /* fast path not storing raddr and rkey */ + mpi_errno = MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, write_to_buf); /* fast path not storing raddr and rkey */ if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -280,8 +280,7 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); @@ -292,10 +291,9 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG); + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG); } - if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && - MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) { + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) { dprintf("lmt_start_recv,send_progress\n"); fflush(stdout); MPID_nem_ib_send_progress(vc_ib); @@ -360,7 +358,8 @@ int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) } //assert(dt_true_lb == 0); - uint8_t *tailp = (uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t)); + uint8_t *tailp = + (uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t)); #if 0 *is_end_flag_same = (r_cookie_buf->tail == *tailp) ? 1 : 0; #else diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 021e354..77d4c48 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -65,7 +65,8 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) #if 0 /*def HAVE_LIBDCFA */ result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, 1, &cqe[0]); #else - result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, /*3 */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = + ibv_poll_cq(MPID_nem_ib_rc_shared_scq, /*3 */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); #endif MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); @@ -190,7 +191,7 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) < + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG); MPID_Request *sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); @@ -356,7 +357,7 @@ int MPID_nem_ib_drain_scq(int dont_call_progress) vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) < + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG); MPID_Request *sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); int msg_type_sreq = MPIDI_Request_get_msg_type(sreq); @@ -413,7 +414,9 @@ int MPID_nem_ib_drain_scq_lmt_put() #if 0 /*def HAVE_LIBDCFA */ result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, 1, &cqe[0]); #else - result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = + ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, + &cqe[0]); #endif MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); @@ -463,7 +466,7 @@ int MPID_nem_ib_drain_scq_lmt_put() MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); vc_ib->ibcom->ncom_lmt_put -= 1; MPID_nem_ib_ncqe_lmt_put -= 1; - dprintf("drain_scq_lmt_put,rndv,ncqe=%d\n", MPID_nem_ib_ncqe_lmt_put); /*suspicious */ + dprintf("drain_scq_lmt_put,rndv,ncqe=%d\n", MPID_nem_ib_ncqe_lmt_put); /*suspicious */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); (VC_FIELD(req->ch.vc, pending_sends)) -= 1; @@ -523,7 +526,9 @@ int MPID_nem_ib_drain_scq_scratch_pad() #if 0 /*def HAVE_LIBDCFA */ result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, 1, &cqe[0]); #else - result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = + ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, + &cqe[0]); #endif MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); @@ -580,7 +585,8 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) //dprintf("ib_poll,ld,rsr_seq_num_poll=%d\n", vc_ib->ibcom->rsr_seq_num_poll); volatile void *buf = (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + - MPID_NEM_IB_COM_RDMABUF_SZSEG * ((uint32_t) vc_ib->ibcom->rsr_seq_num_poll % MPID_NEM_IB_COM_RDMABUF_NSEG); + MPID_NEM_IB_COM_RDMABUF_SZSEG * ((uint32_t) vc_ib->ibcom->rsr_seq_num_poll % + MPID_NEM_IB_COM_RDMABUF_NSEG); volatile MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf; if (sz_hdrmagic->magic != MPID_NEM_IB_COM_MAGIC) { goto fn_exit; @@ -599,7 +605,8 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) int sz_data_pow2; MPID_NEM_IB_SZ_DATA_POW2(sz_hdrmagic->sz); - volatile MPID_nem_ib_tailmagic_t *tailmagic = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + sz_data_pow2); + volatile MPID_nem_ib_tailmagic_t *tailmagic = + (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + sz_data_pow2); dprintf("poll,sz_data_pow2=%d,tailmagic=%p,sz=%d\n", sz_data_pow2, tailmagic, sz_hdrmagic->sz); int k = 0; //tsce = MPID_nem_ib_rdtsc(); printf("9,%ld\n", tsce - tscs); // 55 for 512-byte @@ -638,7 +645,8 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) #if 1 void *rsi; - for (rsi = (void *) buf; rsi < (uint8_t *) buf + sz_hdrmagic->sz; rsi = (uint8_t *) rsi + 64 * 4) { + for (rsi = (void *) buf; rsi < (uint8_t *) buf + sz_hdrmagic->sz; + rsi = (uint8_t *) rsi + 64 * 4) { #ifdef __MIC__ __asm__ __volatile__ ("movq %0, %%rsi;" @@ -656,9 +664,11 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) } #endif - MPIDI_CH3_Pkt_eager_send_t *pkt = (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t)); + MPIDI_CH3_Pkt_eager_send_t *pkt = + (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t)); MPIU_Assert(sz_hdrmagic->sz >= - sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_tailmagic_t)); + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + + sizeof(MPID_nem_ib_tailmagic_t)); MPIDI_CH3_Pkt_eager_send_t *pkt2 = (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPID_nem_ib_pkt_prefix_t)); @@ -691,16 +701,17 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) int notify_rate; ibcom_errno = - MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, &notify_rate); + MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, + &notify_rate); dprintf("poll_eager,sendq=%d,ncom=%d,ncqe=%d,ldiff=%d(%d-%d),rdiff=%d(%d-%d),rate=%d\n", - MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_sendq_empty(vc_ib->sendq), + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), - vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent, - notify_rate); + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent, notify_rate); //dprintf("ib_poll,current pcc=%d\n", MPIDI_CH3I_progress_completion_count.v); @@ -715,12 +726,14 @@ int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) dprintf("poll_eager,released,type=%d,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM=%d\n", pkt->type, MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM); MPID_nem_ib_recv_buf_released(vc, - (void *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t) + - sizeof(MPIDI_CH3_Pkt_t))); + (void *) ((uint8_t *) buf + + sizeof(MPID_nem_ib_sz_hdrmagic_t) + + sizeof(MPIDI_CH3_Pkt_t))); } else { if (sz_hdrmagic->sz == - sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_tailmagic_t)) { + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + + sizeof(MPID_nem_ib_tailmagic_t)) { if (pkt->type == MPIDI_CH3_PKT_EAGERSHORT_SEND //|| pkt->type == MPIDI_CH3_PKT_GET ) { @@ -1096,8 +1109,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); - dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_ib_myrank, vc->pg_rank, - user_data); + dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_ib_myrank, vc->pg_rank, user_data); #if 1 /* moving from ib_poll */ /* unmark magic */ /* magic is located at MPID_NEM_IB_COM_INLINE_DATA boundary and variable length entails multiple prospective locations for the future use */ @@ -1112,10 +1124,12 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) } MPIU_Assert(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] <= user_data && - user_data < vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ); + user_data < + vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ); unsigned long mod = (unsigned long) ((uint8_t *) user_data - - (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) & (MPID_NEM_IB_COM_RDMABUF_SZSEG - 1); + (uint8_t *) vc_ib->ibcom-> + icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) & (MPID_NEM_IB_COM_RDMABUF_SZSEG - 1); void *buf = (void *) ((uint8_t *) user_data - mod); //dprintf("recv_buf_released,clearing,buf=%p\n", buf); @@ -1130,10 +1144,12 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) offset = offset ? ((((offset + 1) << 1) - 1) > MPID_NEM_IB_MAX_DATA_POW2 ? MPID_NEM_IB_MAX_DATA_POW2 : (((offset + 1) << 1) - - 1)) : 15) { - volatile MPID_nem_ib_tailmagic_t *ptr = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + offset); + 1)) : 15) { + volatile MPID_nem_ib_tailmagic_t *ptr = + (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + offset); MPIU_Assert(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] <= ptr && - ptr < vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ); + ptr < + vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ); ptr->magic = 0 /*0xde */ ; if (offset == sz_data_pow2) { break; @@ -1146,7 +1162,8 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) /* mark that one eager-send RDMA-write-to buffer has been released */ int index_slot = (unsigned long) ((uint8_t *) user_data - - (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) / MPID_NEM_IB_COM_RDMABUF_SZSEG; + (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) / + MPID_NEM_IB_COM_RDMABUF_SZSEG; MPIU_Assert(0 <= index_slot && index_slot < MPID_NEM_IB_COM_RDMABUF_NSEG); //dprintf("user_data=%p,mem=%p,sub=%08lx,index_slot=%d\n", user_data, vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], (unsigned long)user_data - (unsigned long)vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], index_slot); //dprintf("index_slot=%d,released=%016lx\n", index_slot, vc_ib->ibcom->rsr_seq_num_released[index_slot / 64]); @@ -1158,11 +1175,10 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) //dprintf("released=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_tail / 64]); if (1 || (index_tail & 7) || MPID_nem_ib_diff32(index_slot, index_tail) >= MPID_NEM_IB_COM_RDMABUF_NSEG - 8) { /* avoid wrap-around */ while (1) { - if (((vc_ib->ibcom-> - rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) { + if (((vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 1) == + 1) { vc_ib->ibcom->rsr_seq_num_tail += 1; - vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= - ~(1ULL << (index_tail & 63)); + vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= ~(1ULL << (index_tail & 63)); dprintf("rsr_seq_num_tail,incremented to %d\n", vc_ib->ibcom->rsr_seq_num_tail); } else { @@ -1174,8 +1190,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) if (((vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 0xff) == 0xff) { vc_ib->ibcom->rsr_seq_num_tail += 8; - vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= - ~(0xffULL << (index_tail & 63)); + vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= ~(0xffULL << (index_tail & 63)); //dprintf("released[index_tail/64]=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_tail / 64]); } } @@ -1184,7 +1199,8 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) int notify_rate; ibcom_errno = - MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, &notify_rate); + MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, + &notify_rate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get"); @@ -1201,7 +1217,8 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) int msg_type = MPIDI_Request_get_msg_type(sreq); if (msg_type == MPIDI_REQUEST_EAGER_MSG && /* guard for the following pointer dereference */ ((MPIDI_CH3_Pkt_t *) sreq->dev.iov[0].MPID_IOV_BUF)->type == MPIDI_NEM_PKT_NETMOD && - ((MPID_nem_pkt_netmod_t *) sreq->dev.iov[0].MPID_IOV_BUF)->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { + ((MPID_nem_pkt_netmod_t *) sreq->dev.iov[0].MPID_IOV_BUF)->subtype == + MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { goto skip; } } @@ -1226,7 +1243,7 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -1273,8 +1290,8 @@ int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */) { MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_eager_send_t *ch3_pkt = @@ -1303,20 +1320,18 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, int *lsr_seq_num_tail; MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_lsr_seq_num_tail_get"); dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail=%d,netmod_pkt->seq_num_tail=%d\n", *lsr_seq_num_tail, netmod_pkt->seq_num_tail); *lsr_seq_num_tail = MPID_NEM_IB_MAX(*lsr_seq_num_tail, netmod_pkt->seq_num_tail); - dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail updated to %d\n", - *lsr_seq_num_tail); + dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail updated to %d\n", *lsr_seq_num_tail); #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,eagersend,old rstate=%d\n", - vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + dprintf("pkthandler,eagersend,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, lsr_seq_num_tail); - dprintf("pkthandler,eagersend,new rstate=%d\n", - vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + dprintf("pkthandler,eagersend,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,eagersend,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", @@ -1367,8 +1382,8 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */) { MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_put_t *ch3_pkt = @@ -1396,8 +1411,7 @@ int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", @@ -1435,8 +1449,8 @@ int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */) { MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_accum_t *ch3_pkt = @@ -1464,8 +1478,7 @@ int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", @@ -1502,8 +1515,8 @@ int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */) { MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_get_t *ch3_pkt = @@ -1531,8 +1544,7 @@ int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", @@ -1568,8 +1580,8 @@ int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */) + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */) { MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_get_t *ch3_pkt = @@ -1597,8 +1609,7 @@ int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, &vc_ib->ibcom->lsr_seq_num_tail); dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", @@ -1633,7 +1644,7 @@ int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -1671,7 +1682,7 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* change remote notification policy of RDMA-write-to buf */ //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, - &vc_ib->ibcom->lsr_seq_num_tail); + &vc_ib->ibcom->lsr_seq_num_tail); //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); @@ -1682,8 +1693,9 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG, - vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail) < + MPID_NEM_IB_COM_RDMABUF_NSEG, vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail); } dprintf("get_done_handler,send_progress\n"); fflush(stdout); @@ -1711,7 +1723,7 @@ int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -1737,7 +1749,7 @@ int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG, + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG, vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail); /* send reply */ @@ -1756,7 +1768,7 @@ int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -1778,7 +1790,8 @@ int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, dprintf("pkthandler,reply_seq_num,old lsr_seq_num=%d,reply_pkt->seq_num_tail=%d\n", vc_ib->ibcom->lsr_seq_num_tail, reply_pkt->seq_num_tail); ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_lsr_seq_num_tail_get"); *lsr_seq_num_tail = MPID_NEM_IB_MAX(*lsr_seq_num_tail, reply_pkt->seq_num_tail); //dprintf("pkthandler,reply_seq_num,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail); @@ -1793,8 +1806,8 @@ int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* try to send from sendq because at least one RDMA-write-to buffer has been released */ //dprintf("pkthandler,reply_seq_num,send_progress\n"); dprintf("pkthandler,reply_seq_num,send_progress\n"); - MPID_NEM_IB_CHECK_AND_SEND_PROGRESS fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM); + MPID_NEM_IB_CHECK_AND_SEND_PROGRESS fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM); return mpi_errno; fn_fail: goto fn_exit; @@ -1805,9 +1818,9 @@ int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, - MPID_Request ** rreqp) + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, + MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -1831,7 +1844,7 @@ int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc int *rdmabuf_occupancy_notify_lstate; ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get(vc_ib->sc->fd, - &rdmabuf_occupancy_notify_lstate); + &rdmabuf_occupancy_notify_lstate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get"); *rdmabuf_occupancy_notify_lstate = reply_pkt->state; @@ -1861,7 +1874,9 @@ int MPID_nem_ib_cm_drain_scq() MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ); - result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = + ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, + &cqe[0]); MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { @@ -1922,23 +1937,26 @@ int MPID_nem_ib_cm_poll() dprintf("cm_poll,enter\n"); - volatile uint32_t *owner = (uint32_t *) (MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO]); + volatile uint32_t *owner = + (uint32_t *) (MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO]); if (*owner == (uint32_t) - 1) { goto fn_exit; } /* not acquired */ MPID_nem_ib_com *ibcom_scratch_pad; - ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[*owner], &ibcom_scratch_pad); + ibcom_errno = + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[*owner], &ibcom_scratch_pad); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); MPID_nem_ib_cm_cmd_t *received = (MPID_nem_ib_cm_cmd_t *) (ibcom_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + - sizeof(uint32_t)); + sizeof(uint32_t)); MPID_nem_ib_cm_cmd_t cmd; MPID_nem_ib_vc_area *vc_ib; switch (received->type) { case MPID_NEM_IB_CM_SYN: - ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[*owner].fd); + ibcom_errno = + MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[*owner].fd); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); cmd.type = MPID_NEM_IB_CM_SYNACK; goto common_tail; @@ -1946,38 +1964,49 @@ int MPID_nem_ib_cm_poll() case MPID_NEM_IB_CM_BUSINESSCARD: ibcom_errno = MPID_nem_ib_com_rts(MPID_nem_ib_conns[*owner].fd, received->qpnum, received->lid, - &(received->gid)); + &(received->gid)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); ibcom_errno = - MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[*owner].fd, received->rmem, received->rkey); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect"); + MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[*owner].fd, received->rmem, + received->rkey); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_reg_mr_connect"); VC_FIELD(MPID_nem_ib_conns[owner].vc, is_connected) = 1; cmd.type = MPID_NEM_IB_CM_ACK; common_tail: ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &(cmd.lid), - sizeof(uint16_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, + MPID_NEM_IB_COM_INFOKEY_PORT_LID, &(cmd.lid), + sizeof(uint16_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &(cmd.gid), - sizeof(union ibv_gid)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, + MPID_NEM_IB_COM_INFOKEY_PORT_GID, &(cmd.gid), + sizeof(union ibv_gid)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &(cmd.qpnum), - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, + MPID_NEM_IB_COM_INFOKEY_QP_QPN, &(cmd.qpnum), + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_SCRATCH_PAD_TO, - MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &(cmd.rmem), sizeof(void *)); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, + MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &(cmd.rmem), + sizeof(void *)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_SCRATCH_PAD_TO, - MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &(cmd.rkey), sizeof(int)); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, + MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &(cmd.rkey), sizeof(int)); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); *owner = (uint32_t) - 1; /* release */ @@ -2030,30 +2059,38 @@ int MPID_nem_ib_cm_accept() MPID_nem_ib_conn_pkt_t *rpkt = (MPID_nem_ib_conn_pkt_t *) (rbuf + 40); if (rpkt->type == MPID_NEM_IB_SYN) { - dprintf("accept,%d<-%d,type=%08x\n", MPID_nem_ib_myrank, rpkt->remote_rank, - rpkt->type); + dprintf("accept,%d<-%d,type=%08x\n", MPID_nem_ib_myrank, rpkt->remote_rank, rpkt->type); void *sbuf; ibcom_errno = MPID_nem_ib_com_mem_udwr_from(MPID_nem_ib_conn_ud_fd, &sbuf); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_mem_udwr_from"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_mem_udwr_from"); MPID_nem_ib_conn_pkt_t *spkt = (MPID_nem_ib_conn_pkt_t *) (sbuf + 40); spkt->remote_rank = MPID_nem_ib_myrank; spkt->type = MPID_NEM_IB_SYNACK; ibcom_errno = - MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[rpkt->remote_rank].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, - &spkt->qpn, sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[rpkt->remote_rank].fd, + MPID_NEM_IB_COM_INFOKEY_QP_QPN, &spkt->qpn, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, MPID_NEM_IB_COM_RDMAWR_TO, - MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &spkt->rmem, sizeof(void *)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, + MPID_NEM_IB_COM_RDMAWR_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &spkt->rmem, + sizeof(void *)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_mr"); ibcom_errno = - MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, MPID_NEM_IB_COM_RDMAWR_TO, - MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &spkt->rkey, sizeof(int)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, + MPID_NEM_IB_COM_RDMAWR_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &spkt->rkey, + sizeof(int)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_get_info_mr"); /* kokomade. add udsend_core(synack) */ if (MPID_nem_ib_conn_ibcom->ncom < &&MPID_nem_ib_ncqe_connect <) { diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c index 145ae53..ab9448e 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c @@ -57,11 +57,13 @@ static inline void *aalloc(size_t sz, int id) arena_flist[id] = ((free_list_t *) arena_flist[id])->next; } else { - q = mmap(NULL, MPID_NEM_IB_ROUNDUP64(MPID_NEM_IB_SZARENA * MPID_NEM_IB_NCLUST_SLAB, 4096), PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + q = mmap(NULL, + MPID_NEM_IB_ROUNDUP64(MPID_NEM_IB_SZARENA * MPID_NEM_IB_NCLUST_SLAB, 4096), + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); #if MPID_NEM_IB_NCLUST_SLAB > 1 arena_flist[id] = q + MPID_NEM_IB_SZARENA; - for (p = arena_flist[id]; p < q + (MPID_NEM_IB_NCLUST_SLAB - 1) * MPID_NEM_IB_SZARENA; p += MPID_NEM_IB_SZARENA) { + for (p = arena_flist[id]; p < q + (MPID_NEM_IB_NCLUST_SLAB - 1) * MPID_NEM_IB_SZARENA; + p += MPID_NEM_IB_SZARENA) { ((free_list_t *) p)->next = p + MPID_NEM_IB_SZARENA; } ((free_list_t *) p)->next = 0; @@ -105,7 +107,8 @@ struct MPID_nem_ib_com_reg_mr_cache_entry_t { int refc; }; -static struct MPID_nem_ib_com_reg_mr_listnode_t MPID_nem_ib_com_reg_mr_cache[MPID_NEM_IB_COM_REG_MR_NLINE]; +static struct MPID_nem_ib_com_reg_mr_listnode_t + MPID_nem_ib_com_reg_mr_cache[MPID_NEM_IB_COM_REG_MR_NLINE]; __inline__ int MPID_nem_ib_com_hash_func(char *addr) { @@ -115,7 +118,8 @@ __inline__ int MPID_nem_ib_com_hash_func(char *addr) return (int) v; } -static void MPID_nem_ib_com_reg_mr_insert(struct MPID_nem_ib_com_reg_mr_listnode_t *c, struct MPID_nem_ib_com_reg_mr_listnode_t *e) +static void MPID_nem_ib_com_reg_mr_insert(struct MPID_nem_ib_com_reg_mr_listnode_t *c, + struct MPID_nem_ib_com_reg_mr_listnode_t *e) { struct MPID_nem_ib_com_reg_mr_listnode_t *next; struct MPID_nem_ib_com_reg_mr_listnode_t *prev; @@ -142,7 +146,9 @@ static inline void __lru_queue_display() int i = 0; for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { dprintf("---- hash %d\n", i); - for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next; + for (p = + (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i]. + lru_next; p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i]; p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) { if (p && p->addr) { @@ -161,8 +167,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) #if 0 /* debug */ struct ibv_mr *mr; int ibcom_errno = MPID_nem_ib_com_reg_mr(addr, len, &mr); - printf("mrcache,MPID_nem_ib_com_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len, mr->lkey, - mr->rkey); + printf("mrcache,MPID_nem_ib_com_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len, + mr->lkey, mr->rkey); if (ibcom_errno != 0) { goto fn_fail; } @@ -183,8 +189,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) #else void *addr_aligned = (void *) ((unsigned long) addr & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); int len_aligned = - ((((unsigned long) addr + len) - (unsigned long) addr_aligned + MPID_NEM_IB_COM_REG_MR_SZPAGE - - 1) & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); + ((((unsigned long) addr + len) - (unsigned long) addr_aligned + + MPID_NEM_IB_COM_REG_MR_SZPAGE - 1) & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); #endif key = MPID_nem_ib_com_hash_func(addr); @@ -193,12 +199,14 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) //__lru_queue_display(); int way = 0; - for (e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next; + for (e = + (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next; e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[key]; e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) e->lru_next, way++) { //dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next); - if (e->addr <= addr_aligned && (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) { + if (e->addr <= addr_aligned && + (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) { //dprintf //("MPID_nem_ib_com_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n", //e->addr, e->len, e->mr->addr, e->mr->length, addr, len); @@ -224,7 +232,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) afree(victim, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); } - e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t), MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); + e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t), + MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); /* reference counter is used when evicting entry */ e->refc = 1; @@ -242,7 +251,8 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) //key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); /* register to cache */ - MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); + MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], + (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); //__lru_queue_display(); @@ -254,9 +264,12 @@ struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) e->refc++; #if 0 /* disable for debug */ /* move to head of the list */ - if (e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next) { + if (e != + (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key]. + lru_next) { MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *) e); - MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); + MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], + (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); } #endif //dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, @@ -291,8 +304,10 @@ void MPID_nem_ib_com_register_cache_init() /* Using the address to the start node to express the end of the list * instead of using NULL */ for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { - MPID_nem_ib_com_reg_mr_cache[i].lru_next = (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i]; - MPID_nem_ib_com_reg_mr_cache[i].lru_prev = (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i]; + MPID_nem_ib_com_reg_mr_cache[i].lru_next = + (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i]; + MPID_nem_ib_com_reg_mr_cache[i].lru_prev = + (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i]; } dprintf("[MrCache] cache initializes %d entries\n", MPID_NEM_IB_COM_REG_MR_NLINE); @@ -304,7 +319,9 @@ void MPID_nem_ib_com_register_cache_destroy() int i = 0, cnt = 0; for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { - for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next; + for (p = + (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i]. + lru_next; p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i]; p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) { if (p && p->addr > 0) { diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index c0ca7e0..48b9b83 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -54,7 +54,8 @@ void MPID_nem_ib_cm_map_set(MPID_nem_ib_cm_map_t * map, char *key, int key_lengt } int s1_minus_s2; while (1) { - int lmin = key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); + int lmin = + key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree); s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin); @@ -66,24 +67,24 @@ void MPID_nem_ib_cm_map_set(MPID_nem_ib_cm_map_t * map, char *key, int key_lengt else if (s1_minus_s2 < 0 || !s1_minus_s2 && residual < 0) { // psArg is "smaller" OR same substring, psArg is shorter if (MPID_NEM_IB_MAP_LPTR(pTree) == 0) { - MPID_NEM_IB_MAP_LPTR(pTree) = map->length; // pointer write + MPID_NEM_IB_MAP_LPTR(pTree) = map->length; // pointer write /* left child */ MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); dprintf("stored as left child\n"); return; } - pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child + pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child } else { // psArg is "larger" OR same substring, psArg is longer if (MPID_NEM_IB_MAP_RPTR(pTree) == 0) { - MPID_NEM_IB_MAP_RPTR(pTree) = map->length; // pointer write + MPID_NEM_IB_MAP_RPTR(pTree) = map->length; // pointer write /* right child */ MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); dprintf("stored as right child\n"); return; } - pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child + pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child } } } @@ -102,7 +103,8 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length } int s1_minus_s2; while (1) { - int lmin = key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); + int lmin = + key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree); s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin); @@ -118,7 +120,7 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length dprintf("left is null\n"); goto fn_fail; } - pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child + pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child } else { // psArg is "larger" OR same substring, psArg is longer @@ -127,7 +129,7 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length dprintf("right is null\n"); goto fn_fail; } - pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child + pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child } } fn_exit: @@ -142,7 +144,7 @@ int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, - MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) + MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -159,7 +161,8 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo /* remote SR sequence number which is last sent */ int *rsr_seq_num_tail; ibcom_errno = MPID_nem_ib_com_rsr_seq_num_tail_get(vc_ib->sc->fd, &rsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rsr_seq_num_tail_get"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_rsr_seq_num_tail_get"); /* remote SR sequence number which is last sent */ int *rsr_seq_num_tail_last_sent; @@ -256,14 +259,14 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo int copied; ibcom_errno = - MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, netmod_hdr, sz_netmod_hdr, hdr, hdr_sz, data, - (int) data_sz, &copied); + MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, netmod_hdr, sz_netmod_hdr, hdr, + hdr_sz, data, (int) data_sz, &copied); MPIU_ERR_CHKFATALANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_isend"); MPID_nem_ib_ncqe += 1; //dprintf("isendcontig_core,ncqe=%d\n", MPID_nem_ib_ncqe); dprintf("isendcontig_core,isend,kind=%d,msg_type=%d,copied=%d\n", sreq->kind, msg_type, copied); /*suspicious lines,(if1,on,on,off,if0) works */ #if 0 -#define MPID_NEM_IB_TLBPREF_SEND 20 //20 +#define MPID_NEM_IB_TLBPREF_SEND 20 //20 int tlb_pref_ahd = 4096 * MPID_NEM_IB_TLBPREF_SEND; __asm__ __volatile__ ("movq %0, %%rsi;" @@ -336,16 +339,17 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo } } else { - MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ } #else - MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ #endif #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE //dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); int *notify_rstate; - ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &notify_rstate); + ibcom_errno = + MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &notify_rstate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get"); @@ -354,13 +358,14 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo /* if the number of slots in RMDA-write-to buffer have hit the high water-mark */ if (*notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW && MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) > MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) { + vc_ib->ibcom->lsr_seq_num_tail) > + MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) { dprintf("changing notify_rstate,id=%d\n", vc_ib->ibcom->sseq_num); /* remember remote notifying policy so that local can know when to change remote policy back to LW */ *notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW; /* change remote notifying policy of RDMA-write-to buf occupancy info */ MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc, - MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); + MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); } //dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif @@ -377,7 +382,7 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, - MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) + MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -399,10 +404,12 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, #if 0 /* aggressively perform drain_scq */ /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { + if (vc_ib->ibcom->ncom >= + /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || + MPID_nem_ib_ncqe >= + /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { //printf("isendcontig,kick drain_scq\n"); - ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ + ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); } #endif @@ -425,7 +432,8 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, int *lsr_seq_num_tail; /* sequence number of (largest) completed send command */ ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_lsr_seq_num_tail_get"); int lsr_seq_num_head; /* sequence number of (largest) in-flight send command */ @@ -438,7 +446,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail_last_sent), vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent); dprintf("isendcontig,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, @@ -459,7 +467,8 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, /* make control packet bringing sequence number go ahead of * queued packets to avoid dead-lock */ int goahead = - (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REQ_SEQ_NUM) + (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && + netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REQ_SEQ_NUM) || (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) || (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && @@ -474,7 +483,8 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, (goahead || MPID_nem_ib_sendq_empty(vc_ib->sendq)) && vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack && - MPID_nem_ib_diff32(lsr_seq_num_head, *lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { + MPID_nem_ib_diff32(lsr_seq_num_head, + *lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { mpi_errno = MPID_nem_ib_iSendContig_core(vc, sreq, hdr, hdr_sz, data, data_sz); if (mpi_errno) { @@ -526,7 +536,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, dprintf("isendcontig,enqueuing,type=%d,\n", ((MPIDI_CH3_Pkt_t *) hdr)->type); #if 0 if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_NETMOD) { - switch(((MPID_nem_pkt_netmod_t *) hdr)->subtype) { + switch (((MPID_nem_pkt_netmod_t *) hdr)->subtype) { case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM: dprintf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_ncqe); @@ -534,8 +544,9 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, default: break; } - } else { - switch(((MPIDI_CH3_Pkt_t *) hdr)->type) { + } + else { + switch (((MPIDI_CH3_Pkt_t *) hdr)->type) { case MPIDI_CH3_PKT_ACCUMULATE: dprintf("enqueuing ACCUMULATE\n"); break; @@ -581,7 +592,7 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, - void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr) + void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr) { MPID_Request *sreq = NULL; int mpi_errno = MPI_SUCCESS; @@ -649,7 +660,7 @@ int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_s #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, - MPIDI_msg_sz_t hdr_sz) + MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -677,14 +688,15 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, } ibcom_errno = MPID_nem_ib_com_sseq_num_get(vc_ib->sc->fd, &sseq_num); - MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_sseq_num_get"); + MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_com_sseq_num_get"); int copied; dprintf("sendnoncontig_core,isend,%d->%d,seq_num=%d\n", MPID_nem_ib_myrank, vc->pg_rank, sseq_num); ibcom_errno = MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, NULL, 0, hdr, sizeof(MPIDI_CH3_Pkt_t), - (void *) REQ_FIELD(sreq, lmt_pack_buf), (int) last, &copied); + (void *) REQ_FIELD(sreq, lmt_pack_buf), (int) last, &copied); MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_isend"); MPID_nem_ib_ncqe += 1; dprintf("sendnoncontig_core,ncqe=%d\n", MPID_nem_ib_ncqe); @@ -727,17 +739,18 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, } } else { - MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ } #else - MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ #endif #ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE #if 1 //dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); int *notify_rstate; - ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &notify_rstate); + ibcom_errno = + MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &notify_rstate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get"); @@ -746,13 +759,14 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, /* if the number of slots in RMDA-write-to buffer have hit the high water-mark */ if (*notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW && MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) > MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) { + vc_ib->ibcom->lsr_seq_num_tail) > + MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) { dprintf("changing notify_rstate,id=%d\n", vc_ib->ibcom->sseq_num); /* remember remote notifying policy so that local can know when to change remote policy back to LW */ *notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW; /* change remote notifying policy of RDMA-write-to buf occupancy info */ MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc, - MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); + MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); } //dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif @@ -770,7 +784,7 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, - MPIDI_msg_sz_t hdr_sz) + MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -789,10 +803,12 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, #if 0 /* aggressively perform drain_scq */ /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { + if (vc_ib->ibcom->ncom >= + /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || + MPID_nem_ib_ncqe >= + /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { //printf("isendcontig,kick drain_scq\n"); - ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ + ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); } #endif @@ -801,12 +817,12 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, /* if IB command overflow-queue is empty AND local IB command queue isn't full AND remote RDMA-write-to buf isn't getting overrun */ /* set it for drain_scq */ - int slack = MPID_NEM_IB_COM_AMT_SLACK; /* slack for control packet bringing sequence number */ + int slack = MPID_NEM_IB_COM_AMT_SLACK; /* slack for control packet bringing sequence number */ if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack && MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { mpi_errno = MPID_nem_ib_SendNoncontig_core(vc, sreq, hdr, hdr_sz); if (mpi_errno) { @@ -871,10 +887,12 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) #if 0 /* aggressively perform drain_scq */ /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { + if (vc_ib->ibcom->ncom >= + /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || + MPID_nem_ib_ncqe >= + /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { dprintf("send_progress,kick drain_scq\n"); - ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ + ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); } @@ -892,11 +910,13 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) (ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || netmod_hdr->subtype != MPIDI_NEM_IB_PKT_LMT_GET_DONE) && ch3_hdr->type != MPIDI_NEM_PKT_LMT_RTS && - ch3_hdr->type != MPIDI_NEM_PKT_LMT_CTS) ? MPID_NEM_IB_COM_AMT_SLACK : 0) : MPID_NEM_IB_COM_AMT_SLACK; + ch3_hdr->type != + MPIDI_NEM_PKT_LMT_CTS) ? MPID_NEM_IB_COM_AMT_SLACK : 0) : + MPID_NEM_IB_COM_AMT_SLACK; if (vc_ib->ibcom->ncom >= MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack || MPID_nem_ib_ncqe >= MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack || MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail) >= + vc_ib->ibcom->lsr_seq_num_tail) >= MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { break; } @@ -912,10 +932,10 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) dprintf("send_progress,%d->%d,rdiff=%d(%d-%d),ldiff=%d(%d-%d),slack=%d\n", MPID_nem_ib_myrank, sreq->ch.vc->pg_rank, MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, - vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail_last_sent), vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, - vc_ib->ibcom->lsr_seq_num_tail), + vc_ib->ibcom->lsr_seq_num_tail), vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, slack); if (sreq->kind == MPID_REQUEST_SEND && msg_type == MPIDI_REQUEST_EAGER_MSG) { if (!sreq->ch.noncontig) { @@ -969,8 +989,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) * might have happened while being queued */ case MPIDI_NEM_IB_PKT_LMT_GET_DONE:{ MPID_nem_ib_pkt_lmt_get_done_t *_done_pkt = - (MPID_nem_ib_pkt_lmt_get_done_t *) sreq->dev.iov[0]. - MPID_IOV_BUF; + (MPID_nem_ib_pkt_lmt_get_done_t *) sreq->dev. + iov[0].MPID_IOV_BUF; dprintf ("send_progress,MPIDI_NEM_IB_PKT_LMT_GET_DONE,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); @@ -983,8 +1003,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) } case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:{ MPID_nem_ib_pkt_reply_seq_num_t *_pkt = - (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.iov[0]. - MPID_IOV_BUF; + (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev. + iov[0].MPID_IOV_BUF; dprintf ("send_progress,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); @@ -1003,10 +1023,10 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) mpi_errno = MPID_nem_ib_iSendContig_core(sreq->ch.vc, sreq, - sreq->dev.iov[0].MPID_IOV_BUF, - sreq->dev.iov[0].MPID_IOV_LEN, - sreq->dev.iov[1].MPID_IOV_BUF, - sreq->dev.iov[1].MPID_IOV_LEN); + sreq->dev.iov[0].MPID_IOV_BUF, + sreq->dev.iov[0].MPID_IOV_LEN, + sreq->dev.iov[1].MPID_IOV_BUF, + sreq->dev.iov[1].MPID_IOV_LEN); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1015,8 +1035,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) dprintf("send_progress,send,noncontig\n"); mpi_errno = MPID_nem_ib_SendNoncontig_core(sreq->ch.vc, sreq, - sreq->dev.iov[0].MPID_IOV_BUF, - sreq->dev.iov[0].MPID_IOV_LEN); + sreq->dev.iov[0].MPID_IOV_BUF, + sreq->dev.iov[0].MPID_IOV_LEN); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1028,8 +1048,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) MPID_nem_ib_sendq_next(sreq)); mpi_errno = MPID_nem_ib_lmt_start_recv_core(sreq, REQ_FIELD(sreq, lmt_raddr), - REQ_FIELD(sreq, lmt_rkey), REQ_FIELD(sreq, - lmt_write_to_buf)); + REQ_FIELD(sreq, lmt_rkey), REQ_FIELD(sreq, + lmt_write_to_buf)); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -1079,7 +1099,8 @@ int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) int MPID_nem_ib_cm_send_core(int rank, MPID_nem_ib_cm_cmd_t * cmd) { MPID_nem_ib_com *ibcom_scratch_pad; - ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[rank], &ibcom_scratch_pad); + ibcom_errno = + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[rank], &ibcom_scratch_pad); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); if (MPID_nem_ib_ncqe_scratch_pad >= MPID_NEM_IB_COM_MAX_SQ_CAPACITY || @@ -1090,8 +1111,9 @@ int MPID_nem_ib_cm_send_core(int rank, MPID_nem_ib_cm_cmd_t * cmd) } ibcom_errno = - MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) ibcom_scratch_pad, - sizeof(uint32_t), sizeof(MPID_nem_ib_cm_cmd_t), (void *) cmd); + MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], + (uint64_t) ibcom_scratch_pad, sizeof(uint32_t), + sizeof(MPID_nem_ib_cm_cmd_t), (void *) cmd); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_put_scratch_pad"); MPID_nem_ib_ncqe_scratch_pad += 1; ----------------------------------------------------------------------- Summary of changes: src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c | 474 +++++++++++++------- src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h | 94 ++-- src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h | 84 ++-- src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c | 223 ++++++---- src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c | 21 +- src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c | 253 ++++++----- .../ch3/channels/nemesis/netmod/ib/ib_reg_mr.c | 55 ++- src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c | 154 ++++--- 8 files changed, 799 insertions(+), 559 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-89-g37012b6
by noreply＠mpich.org 19 Dec '13

19 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 37012b672626a5113b2d5707d8c8af6a850de71e (commit) via 0cbef04f3a515b323542105b6ce4a8e43bf026b0 (commit) via 17495f90a8d0b7c480618c26dc6bf2b850799be0 (commit) via 0ab7dc738033768194bdcbddb8915826cd409c95 (commit) via 88c06bf434fd36c283b30495512cd9db6ea63dd2 (commit) via 47c0ca0fea88063da89d97e3c5425db94175f344 (commit) via b9f5aa8fd45accaeefe9c4e2ba2ad6852644bad0 (commit) via 44f906a47c12b0c1e037fd3374a79606aca002e6 (commit) via e87f7e4e46fab8fc58a3f82f0e986760c1a37b33 (commit) via f6243629523ab6501cceb336d64f2037700d9059 (commit) via e7de1763a0ba7fcddd9ee1ad5aec7961ea281bef (commit) via 0fcd7852884622a8869ed767fd634423d5092523 (commit) from 5711e32d3597f48bc57d092d30b40950134fc40c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/37012b672626a5113b2d5707d8c8af6a8… commit 37012b672626a5113b2d5707d8c8af6a850de71e Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Fri Dec 13 17:55:47 2013 +0900 Modify README.vin on description of the IB netmod Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/README.vin b/README.vin index 218500c..d436a8e 100644 --- a/README.vin +++ b/README.vin @@ -474,26 +474,26 @@ shared memory has limits on the size of shared memory segments so using this for Nemesis may limit the number of processes that can be started on a single node. -dcfa network module -``````````````````` -The DCFA netmod provides support for InfiniBand on x86_64 platforms -(including Xeon Phi). It can be built in the following +ib network module +````````````````` +The IB netmod provides support for InfiniBand on x86_64 platforms +(including Xeon Phi). It can be built in the following configurations: 1. InfiniBand Open Fabrics, x86_64 (host), Linux For this mode, configure with the following option: - --with-device=ch3:nemesis:dcfa + --with-device=ch3:nemesis:ib 2. InfiniBand Open Fabrics, Xeon Phi, Intel MPSS Linux For this mode, configure with the following options: - --with-device=ch3:nemesis:dcfa - --with-cross=<mpich-source>/src/mpid/ch3/channels/nemesis/netmod/dcfa/cross_values.txt + --with-device=ch3:nemesis:ib + --with-cross=<mpich-source>/src/mpid/ch3/channels/nemesis/netmod/ib/cross_values.txt --host=x86_64-k1om-linux - --with-dcfa=/opt/intel/mic/ofed/card/usr + --with-ib=/opt/intel/mic/ofed/card/usr CC=icc CXX=icpc F77=ifort FC=ifort CFLAGS=-mmic CXXFLAGS=-mmic FFLAGS=-mmic FCFLAGS=-mmic LDFLAGS=-mmic @@ -502,10 +502,10 @@ configurations: For this mode, configure with the following options: - --with-device=ch3:nemesis:dcfa - --with-cross=<mpich-source>/src/mpid/ch3/channels/nemesis/netmod/dcfa/cross_values.txt + --with-device=ch3:nemesis:ib + --with-cross=<mpich-source>/src/mpid/ch3/channels/nemesis/netmod/ib/cross_values.txt --host=x86_64-k1om-linux - --with-dcfa=<mckernel-source>/attached + --with-ib=<mckernel-source>/attached --disable-shared CC=icc CXX=icpc F77=ifort FC=ifort CFLAGS=-mmic CXXFLAGS=-mmic FFLAGS=-mmic FCFLAGS=-mmic LDFLAGS=-mmic http://git.mpich.org/mpich.git/commitdiff/0cbef04f3a515b323542105b6ce4a8e43… commit 0cbef04f3a515b323542105b6ce4a8e43bf026b0 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Fri Dec 13 15:18:19 2013 +0900 Correct references to "subtype" of netmod packets Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 958a4d6..021e354 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -1199,9 +1199,9 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); if (sreq) { int msg_type = MPIDI_Request_get_msg_type(sreq); - MPIDI_CH3_Pkt_t *ch3_hdr = (MPIDI_CH3_Pkt_t *) sreq->dev.iov[0].MPID_IOV_BUF; if (msg_type == MPIDI_REQUEST_EAGER_MSG && /* guard for the following pointer dereference */ - ch3_hdr->type == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { + ((MPIDI_CH3_Pkt_t *) sreq->dev.iov[0].MPID_IOV_BUF)->type == MPIDI_NEM_PKT_NETMOD && + ((MPID_nem_pkt_netmod_t *) sreq->dev.iov[0].MPID_IOV_BUF)->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { goto skip; } } diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index 5ca2f82..c0ca7e0 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -525,27 +525,39 @@ int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, */ dprintf("isendcontig,enqueuing,type=%d,\n", ((MPIDI_CH3_Pkt_t *) hdr)->type); #if 0 - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { - printf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_ib_myrank, vc->pg_rank, - MPID_nem_ib_ncqe); - } - //if (((MPIDI_CH3_Pkt_t *)hdr)->type == MPIDI_CH3_PKT_ACCUMULATE) { - //printf("enqueuing ACCUMULATE\n"); - //} - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_GET_RESP) { - printf("enqueuing GET_RESP\n"); - } - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_GET) { - printf("enqueuing GET\n"); - } - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_CH3_PKT_PUT) { - printf("enqueuing PUT\n"); + if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_NETMOD) { + switch(((MPID_nem_pkt_netmod_t *) hdr)->subtype) { + case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM: + dprintf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_ib_myrank, vc->pg_rank, + MPID_nem_ib_ncqe); + break; + default: + break; + } + } else { + switch(((MPIDI_CH3_Pkt_t *) hdr)->type) { + case MPIDI_CH3_PKT_ACCUMULATE: + dprintf("enqueuing ACCUMULATE\n"); + break; + case MPIDI_CH3_PKT_GET_RESP: + dprintf("enqueuing GET_RESP\n"); + break; + case MPIDI_CH3_PKT_GET: + dprintf("enqueuing GET\n"); + break; + case MPIDI_CH3_PKT_PUT: + dprintf("enqueuing PUT\n"); + break; + case MPIDI_NEM_PKT_LMT_DONE: + dprintf("isendcontig,enqueue,DONE\n"); + break; + default: + break; + } } #endif - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_LMT_DONE) { - dprintf("isendcontig,enqueue,DONE\n"); - } - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { + if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_NETMOD && + ((MPID_nem_pkt_netmod_t *) hdr)->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { dprintf("isendcontig,REPLY_SEQ_NUM,enqueue_at_head\n"); MPID_nem_ib_sendq_enqueue_at_head(&vc_ib->sendq, sreq); } http://git.mpich.org/mpich.git/commitdiff/17495f90a8d0b7c480618c26dc6bf2b85… commit 17495f90a8d0b7c480618c26dc6bf2b850799be0 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Fri Dec 13 12:08:46 2013 +0900 Remove obsolete code Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c index ce2db17..3fae51f 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -30,8 +30,6 @@ #define dprintf(...) #endif -int dflag; - static int sendwr_id = 10; static MPID_nem_ib_com_t contab[MPID_NEM_IB_COM_SIZE]; static int ib_initialized = 0; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h index fcd1ea3..e77091e 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h @@ -346,9 +346,7 @@ extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(int condesc, int *n extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(int condesc, int **rstate); extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get(int condesc, int **lstate); -extern int MPID_nem_ib_comMemInfo(int, int, void **, size_t *, int *); extern char *MPID_nem_ib_com_strerror(int errno); -extern int dflag; extern int MPID_nem_ib_com_mem_rdmawr_from(int condesc, void **out); extern int MPID_nem_ib_com_mem_rdmawr_to(int condesc, int seq_num, void **out); @@ -360,7 +358,6 @@ extern void MPID_nem_ib_com_register_cache_init(void); extern void MPID_nem_ib_com_register_cache_destroy(void); extern struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len); -/* dcfa_ctlmsg.c */ extern int MPID_nem_ib_com_udbuf_init(void *q); #define MPID_NEM_IB_COM_RC_SHARED_RCQ 0 diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h index 1be6a25..aaddb5c 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -325,9 +325,6 @@ int MPID_nem_ib_lmt_vc_terminated(struct MPIDI_VC *vc); int MPID_nem_ib_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req); int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data); -void pht_update(uint64_t vpc, uint32_t hist, int32_t taken); -int pht_pred(uint64_t vpc, uint32_t hist); - /* Keys for business cards */ #define MPID_NEM_IB_GID_KEY "gid" #define MPID_NEM_IB_LID_KEY "lid" diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 4552f51..f0c6ea4 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -425,28 +425,6 @@ int MPID_nem_ib_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req) MPIU_Free(req->ch.s_cookie); //dprintf("lmt_done_send,free cookie,%p\n", req->ch.s_cookie); -#ifdef LMT_PRED - if (MPID_rndv_pred_nlearn == 1 || MPID_rndv_pred_npractice > 1) { - if (req->dev.rndv_pred_decision == MPIDI_CH3_RNDV_SEND_RTS) { - MPID_rndv_pred_hit++; - MPID_rndv_pred_count++; - } - } - if (MPID_rndv_pred_nlearn > 1) { - MPID_hist = (MPID_hist << 1) | 1; - pht_update((uint64_t) buf, MPID_hist, 1); - MPID_rndv_pred_nlearn++; - if (MPID_rndv_pred_nlearn > MPID_RNDV_PRED_MAXLEARN) { - MPID_rndv_pred_nlearn = 0; - MPID_rndv_pred_npractice = 1; - } - } - if (MPID_rndv_pred_nlearn == 1) { - MPID_rndv_pred_nlearn++; - } -#endif - - /* free temporal buffer for eager-send non-contiguous data. * MPIDI_CH3U_Recvq_FDU_or_AEP (in mpid_isend.c) sets req->dev.datatype */ int is_contig; http://git.mpich.org/mpich.git/commitdiff/0ab7dc738033768194bdcbddb8915826c… commit 0ab7dc738033768194bdcbddb8915826cd409c95 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Fri Dec 13 14:24:02 2013 +0900 Replace MPIU_QUOTE with MPIDI_QUOTE Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c index 5b96f5b..25cd464 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c @@ -1005,7 +1005,7 @@ int MPID_nem_ib_vc_destroy(MPIDI_VC_t * vc) #undef FUNCNAME #define FUNCNAME MPID_nem_ib_vc_terminate #undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) +#define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) { dprintf("ib_vc_terminate,enter\n"); diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index c6cd009..958a4d6 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -1911,7 +1911,7 @@ int MPID_nem_ib_cm_drain_scq() #undef FUNCNAME #define FUNCNAME MPID_nem_ib_cm_poll #undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) +#define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_cm_poll() { int mpi_errno = MPI_SUCCESS; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index e61c3cd..5ca2f82 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -830,7 +830,7 @@ int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, #undef FUNCNAME #define FUNCNAME MPID_nem_ib_send_progress #undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) +#define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) { int mpi_errno = MPI_SUCCESS; @@ -1089,7 +1089,7 @@ int MPID_nem_ib_cm_send_core(int rank, MPID_nem_ib_cm_cmd_t * cmd) #undef FUNCNAME #define FUNCNAME MPID_nem_ib_cm_connect #undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) +#define FCNAME MPIDI_QUOTE(FUNCNAME) int MPID_nem_ib_cm_connect(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; http://git.mpich.org/mpich.git/commitdiff/88c06bf434fd36c283b30495512cd9db6… commit 88c06bf434fd36c283b30495512cd9db6ea63dd2 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Thu Dec 12 17:15:34 2013 +0900 Add a header file to Makefile.mk of ib netmod Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk b/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk index 22854b0..de363dc 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk @@ -17,6 +17,7 @@ lib_lib@MPILIBNAME@_la_SOURCES += \ src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c noinst_HEADERS += \ - src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h + src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h endif BUILD_NEMESIS_NETMOD_IB http://git.mpich.org/mpich.git/commitdiff/47c0ca0fea88063da89d97e3c5425db94… commit 47c0ca0fea88063da89d97e3c5425db94175f344 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Thu Dec 12 16:48:17 2013 +0900 Rename the DCFA netmod to the IB netmod Perform git mv files and rename names in files. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk b/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk index 06f653f..ccf83dc 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk +++ b/src/mpid/ch3/channels/nemesis/netmod/Makefile.mk @@ -11,4 +11,4 @@ include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/none/Makefile.mk include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/newmad/Makefile.mk include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/scif/Makefile.mk include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/portals4/Makefile.mk -include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/dcfa/Makefile.mk +include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/Makefile.mk b/src/mpid/ch3/channels/nemesis/netmod/dcfa/Makefile.mk deleted file mode 100644 index f2bea46..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/Makefile.mk +++ /dev/null @@ -1,22 +0,0 @@ -## -*- Mode: Makefile; -*- -## vim: set ft=automake : -## -## (C) 2011 by Argonne National Laboratory. -## See COPYRIGHT in top-level directory. -## - -if BUILD_NEMESIS_NETMOD_DCFA - -lib_lib@MPILIBNAME@_la_SOURCES += \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c - -noinst_HEADERS += \ - src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h - -endif BUILD_NEMESIS_NETMOD_DCFA diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c deleted file mode 100644 index a8c308e..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c +++ /dev/null @@ -1,2217 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * (C) 2012 Oct 14 Yutaka Ishikawa, ishikawa(a)is.s.u-tokyo.ac.jp - * See COPYRIGHT in top-level directory. - */ - -/* - * TODO: - * - ibcomClean might not clean all allocated memory area. Need to FIX it. - * - During error processing in each function, some memory area might not - * be deallocated. Look at all functions. - */ -#include "dcfa_ibcom.h" -#include <sys/ipc.h> -#include <sys/shm.h> -#include <sys/types.h> -#include <assert.h> -#include <linux/mman.h> /* make it define MAP_ANONYMOUS */ -#include <sys/mman.h> - -//#define DEBUG_IBCOM -#ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ -#undef dprintf -#endif -#ifdef DEBUG_IBCOM -#define dprintf printf -#else -#define dprintf(...) -#endif - -int dflag; - -static int sendwr_id = 10; -static IbCom contab[IBCOM_SIZE]; -static int ib_initialized = 0; -static int maxcon; -static struct ibv_device **ib_devlist; -static struct ibv_context *ib_ctx; -struct ibv_context *ib_ctx_export; /* for SC13 demo connector */ -static struct ibv_pd *ib_pd; -struct ibv_pd *ib_pd_export; /* for SC13 demo connector */ -struct ibv_cq *rc_shared_scq; -struct ibv_cq *rc_shared_scq_lmt_put; -struct ibv_cq *rc_shared_scq_scratch_pad; -static struct ibv_cq *rc_shared_rcq; -static struct ibv_cq *rc_shared_rcq_lmt_put; -static struct ibv_cq *rc_shared_rcq_scratch_pad; -static struct ibv_cq *ud_shared_scq; -struct ibv_cq *ud_shared_rcq; -static uint8_t *scratch_pad = 0; - -#define RANGE_CHECK(condesc, conp) \ -{ \ - if (condesc < 0 || condesc >= IBCOM_SIZE) return; \ - conp = &contab[condesc]; \ - if (conp->icom_used != 1) return; \ -} - -#define RANGE_CHECK_WITH_ERROR(condesc, conp) \ -{ \ - if (condesc < 0 || condesc >= IBCOM_SIZE) { \ - return -1; \ - } \ - conp = &contab[condesc]; \ - IBCOM_ERR_CHKANDJUMP(conp->icom_used != 1, -1, dprintf("RANGE_CHECK_WITH_ERROR,conp->icom_used=%d\n", conp->icom_used)); \ -} - -static int modify_qp_to_init(struct ibv_qp *qp, int ib_port) -{ - struct ibv_qp_attr attr; - int flags; - int rc; - - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_INIT; - attr.port_num = ib_port; - attr.pkey_index = 0; - attr.qp_access_flags = - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; - flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS; - rc = ibv_modify_qp(qp, &attr, flags); - if (rc) { - fprintf(stderr, "failed to modify QP state to INIT\n"); - } - return rc; -} - -static int modify_qp_to_rtr(struct ibv_qp *qp, uint32_t remote_qpn, uint16_t dlid, - union ibv_gid *dgid, int ib_port, int gid_idx) -{ - struct ibv_qp_attr attr; - int flags; - int rc; - - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RTR; - attr.path_mtu = IBV_MTU_2048; - //attr.path_mtu = IBV_MTU_1024; - //attr.path_mtu = IBV_MTU_256; /* DCFA */ - attr.dest_qp_num = remote_qpn; - attr.rq_psn = 0; - attr.max_dest_rd_atomic = IBCOM_MAX_RD_ATOMIC; - //attr.max_dest_rd_atomic = 1; - //attr.max_dest_rd_atomic = 0; /* DCFA */ - - /* Default is 0x12 (= 5.12ms) see IB Spec. Rel. 1.2, Vol. 1, 9.7.5.2.8 */ - attr.min_rnr_timer = 0x12; - - attr.ah_attr.dlid = dlid; - attr.ah_attr.sl = 0; - attr.ah_attr.src_path_bits = 0; - attr.ah_attr.is_global = 0; - attr.ah_attr.port_num = ib_port; - - /* In dcfa gid is not set and for testing here it is also not set */ -#if 1 -#ifdef HAVE_LIBDCFA /* DCFA doesn't use gid */ -#else - if (gid_idx >= 0) { - attr.ah_attr.is_global = 1; - attr.ah_attr.port_num = ib_port; - memcpy(&attr.ah_attr.grh.dgid, dgid, 16); - attr.ah_attr.grh.flow_label = 0; - attr.ah_attr.grh.hop_limit = 1; - attr.ah_attr.grh.sgid_index = gid_idx; - attr.ah_attr.grh.traffic_class = 0; - } -#endif -#endif - - flags = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN - | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; - rc = ibv_modify_qp(qp, &attr, flags); - if (rc) { - dprintf("failed to modify QP state to RTR\n"); - } - return rc; -} - -static int modify_qp_to_rts(struct ibv_qp *qp) -{ - struct ibv_qp_attr attr; - int flags; - int rc; - - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RTS; - attr.timeout = (0x14); /* timeout 4.096us * 2^x */ - attr.retry_cnt = 7; - attr.rnr_retry = 7; - attr.sq_psn = 0; - attr.max_rd_atomic = IBCOM_MAX_RD_ATOMIC; - //attr.max_rd_atomic = 1; - //attr.max_rd_atomic = 0; /* DCFA */ - - flags = IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT - | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC; - rc = ibv_modify_qp(qp, &attr, flags); - if (rc) { - fprintf(stderr, "failed to modify QP state to RTS\n"); - } - return rc; -} - -/* called from ibcomOpen if needed */ -static int ibcomDeviceinit() -{ - int ibcom_errno = 0; - int dev_num; - char *dev_name; - int i; - - if (ib_initialized == 1) { - dprintf("ibcomDeviceinit,already initialized\n"); - return 0; - } - if (ib_initialized == -1) - return -1; - - /* initialize ibv_reg_mr cache */ - ibcom_RegisterCacheInit(); - - /* Get the device list */ - ib_devlist = ibv_get_device_list(&dev_num); - if (!ib_devlist || !dev_num) { - fprintf(stderr, "No IB device is found\n"); - return -1; - } - -#ifdef HAVE_LIBDCFA - for (i = 0; i < dev_num; i++) { - if (ib_devlist[i]) { - goto dev_found; - } - } -#else - for (i = 0; i < dev_num; i++) { - if (!strcmp(ibv_get_device_name(ib_devlist[i]), "mlx4_0")) { - goto dev_found; - } - } -#endif - IBCOM_ERR_SETANDJUMP(-1, printf("IB device not found")); - dev_found: - - /* Open the requested device */ - if (ib_ctx_export) { - ib_ctx = ib_ctx_export; - } - else { - ib_ctx = ibv_open_device(ib_devlist[i]); - } - dprintf("ibcomDeviceinit,ib_ctx_export=%p,ib_ctx=%p\n", ib_ctx_export, ib_ctx); - if (!ib_ctx) { - fprintf(stderr, "failed to open IB device\n"); - goto err_exit; - } - ib_ctx_export = ib_ctx; -#ifdef HAVE_LIBDCFA -#else - dev_name = strdup(ibv_get_device_name(ib_devlist[i])); - dprintf("ibcomDeviceinit,dev_name=%s\n", dev_name); -#endif - /* Create a PD */ - if (ib_pd_export) { - ib_pd = ib_pd_export; - } - else { - ib_pd = ibv_alloc_pd(ib_ctx); - } - dprintf("ibcomDeviceinit,ib_pd_export=%p,ib_pd=%p\n", ib_pd_export, ib_pd); - if (!ib_pd) { - fprintf(stderr, "ibv_alloc_pd failed\n"); - goto err_exit; - } - ib_pd_export = ib_pd; - - ib_initialized = 1; - fn_exit: - return ibcom_errno; - - err_exit: - ib_initialized = -1; - if (ib_devlist) - ibv_free_device_list(ib_devlist); - if (ib_ctx) - ibv_close_device(ib_ctx); - return -1; - fn_fail: - goto fn_exit; -} - -static void ibcomClean(IbCom * conp) -{ - int i; - - if (conp->icom_qp) - ibv_destroy_qp(conp->icom_qp); - if (conp->icom_mrlist && conp->icom_mrlen > 0) { - switch (conp->open_flag) { - case IBCOM_OPEN_RC: - for (i = 0; i < IBCOM_NBUF_RDMA; i++) { - if (conp->icom_mrlist[i]) { - ibv_dereg_mr(conp->icom_mrlist[i]); - } - } - break; - case IBCOM_OPEN_SCRATCH_PAD: - for (i = 0; i < IBCOM_NBUF_SCRATCH_PAD; i++) { - if (conp->icom_mrlist[i]) { - ibv_dereg_mr(conp->icom_mrlist[i]); - } - } - break; - case IBCOM_OPEN_UD: - for (i = 0; i < IBCOM_NBUF_UD; i++) { - if (conp->icom_mrlist[i]) { - ibv_dereg_mr(conp->icom_mrlist[i]); - } - } - break; - } - free(conp->icom_mrlist); - } - if (conp->icom_mem[IBCOM_RDMAWR_FROM]) { - munmap(conp->icom_mem[IBCOM_RDMAWR_FROM], IBCOM_RDMABUF_SZ); - } - if (conp->icom_mem[IBCOM_RDMAWR_TO]) { - munmap(conp->icom_mem[IBCOM_RDMAWR_TO], IBCOM_RDMABUF_SZ); - } - if (conp->icom_scq) { - ibv_destroy_cq(conp->icom_scq); - } - if (conp->icom_rcq) { - ibv_destroy_cq(conp->icom_rcq); - } - if (conp->icom_rmem) { - free(conp->icom_rmem); - } - if (conp->icom_rsize) { - free(conp->icom_rsize); - } - if (conp->icom_rkey) { - free(conp->icom_rkey); - } - memset(conp, 0, sizeof(IbCom)); - // TODO: free ah, sge, command template, ... -} - -int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc) -{ - int ibcom_errno = 0, ib_errno; - IbCom *conp; - struct ibv_qp_init_attr qp_init_attr; - struct ibv_sge *sge; - struct ibv_send_wr *sr; - struct ibv_recv_wr *rr, *bad_wr; - int mr_flags; - int i; - - dprintf("ibcomOpen,port=%d,flag=%08x\n", ib_port, ibcom_open_flag); - - int ibcom_open_flag_conn = ibcom_open_flag; - if (ibcom_open_flag_conn != IBCOM_OPEN_RC && - ibcom_open_flag_conn != IBCOM_OPEN_RC_LMT_PUT && - ibcom_open_flag_conn != IBCOM_OPEN_UD && ibcom_open_flag_conn != IBCOM_OPEN_SCRATCH_PAD) { - dprintf("ibcomOpen,bad flag\n"); - ibcom_errno = -1; - goto fn_fail; - } - - /* device open error */ - if (ibcomDeviceinit() < 0) { - ibcom_errno = -1; - goto fn_fail; - } - - /* no more connection can be estabilished */ - if (maxcon == IBCOM_SIZE) { - ibcom_errno = -1; - goto fn_fail; - } - - for (*condesc = 0; *condesc < IBCOM_SIZE; (*condesc)++) { - //dprintf("*condesc=%d,used=%d\n", *condesc, contab[*condesc].icom_used); - if (contab[*condesc].icom_used == 0) { - goto ok_cont; - } - } - /* count says not full, but we couldn't fine vacant slot */ - dprintf("contable has inconsistent\n"); - ibcom_errno = -1; - goto fn_fail; - - ok_cont: - dprintf("ibcomOpen,condesc=%d\n", *condesc); - conp = &contab[*condesc]; - memset(conp, 0, sizeof(IbCom)); - conp->icom_used = 1; - conp->icom_port = ib_port; - conp->open_flag = ibcom_open_flag; - conp->rsr_seq_num_poll = 0; /* it means slot 0 is polled */ - conp->rsr_seq_num_tail = -1; /* it means slot 0 is not released */ - conp->rsr_seq_num_tail_last_sent = -1; - conp->lsr_seq_num_tail = -1; - conp->lsr_seq_num_tail_last_requested = -2; - conp->rdmabuf_occupancy_notify_rstate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW; - conp->rdmabuf_occupancy_notify_lstate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW; - //dprintf("ibcomOpen,ptr=%p,rsr_seq_num_poll=%d\n", conp, conp->rsr_seq_num_poll); - -#ifdef HAVE_LIBDCFA -#else - if (ibv_query_port(ib_ctx, ib_port, &conp->icom_pattr)) { - dprintf("ibv_query_port on port %u failed\n", ib_port); - goto err_exit; - } -#endif - - /* Create send/recv CQ */ - switch (ibcom_open_flag) { - case IBCOM_OPEN_RC: - if (!rc_shared_scq) { -#ifdef HAVE_LIBDCFA - rc_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - rc_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!rc_shared_scq, -1, dprintf("rc_shared_scq")); - } - conp->icom_scq = rc_shared_scq; - - if (!rc_shared_rcq) { -#ifdef HAVE_LIBDCFA - rc_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - rc_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!rc_shared_rcq, -1, dprintf("rc_shared_rcq")); - } - conp->icom_rcq = rc_shared_rcq; - break; - case IBCOM_OPEN_SCRATCH_PAD: - if (!rc_shared_scq_scratch_pad) { -#ifdef HAVE_LIBDCFA - rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!rc_shared_scq_scratch_pad, -1, dprintf("rc_shared_scq")); - } - conp->icom_scq = rc_shared_scq_scratch_pad; - - if (!rc_shared_rcq_scratch_pad) { -#ifdef HAVE_LIBDCFA - rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!rc_shared_rcq_scratch_pad, -1, dprintf("rc_shared_rcq")); - } - conp->icom_rcq = rc_shared_rcq_scratch_pad; - break; - case IBCOM_OPEN_RC_LMT_PUT: - if (!rc_shared_scq_lmt_put) { -#ifdef HAVE_LIBDCFA - rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!rc_shared_scq_lmt_put, -1, dprintf("rc_shared_scq")); - } - conp->icom_scq = rc_shared_scq_lmt_put; - - if (!rc_shared_rcq_lmt_put) { -#ifdef HAVE_LIBDCFA - rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!rc_shared_rcq_lmt_put, -1, dprintf("rc_shared_rcq")); - } - conp->icom_rcq = rc_shared_rcq_lmt_put; - break; - case IBCOM_OPEN_UD: - if (!ud_shared_scq) { -#ifdef HAVE_LIBDCFA - ud_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - ud_shared_scq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!ud_shared_scq, -1, dprintf("ud_shared_scq")); - } - conp->icom_scq = ud_shared_scq; - - if (!ud_shared_rcq) { -#ifdef HAVE_LIBDCFA - ud_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY); -#else - ud_shared_rcq = ibv_create_cq(ib_ctx, IBCOM_MAX_CQ_CAPACITY, NULL, NULL, 0); -#endif - IBCOM_ERR_CHKANDJUMP(!ud_shared_rcq, -1, dprintf("ud_shared_rcq")); - } - conp->icom_rcq = ud_shared_rcq; - break; - } - - /* Create QP */ - memset(&qp_init_attr, 0, sizeof(qp_init_attr)); - qp_init_attr.send_cq = conp->icom_scq; - qp_init_attr.recv_cq = conp->icom_rcq; - qp_init_attr.cap.max_send_wr = IBCOM_MAX_SQ_CAPACITY; - qp_init_attr.cap.max_recv_wr = IBCOM_MAX_RQ_CAPACITY; - qp_init_attr.cap.max_send_sge = IBCOM_MAX_SGE_CAPACITY; - qp_init_attr.cap.max_recv_sge = IBCOM_MAX_SGE_CAPACITY; - qp_init_attr.cap.max_inline_data = IBCOM_INLINE_DATA; - switch (ibcom_open_flag) { - case IBCOM_OPEN_RC: - case IBCOM_OPEN_RC_LMT_PUT: - case IBCOM_OPEN_SCRATCH_PAD: - qp_init_attr.qp_type = IBV_QPT_RC; - break; - case IBCOM_OPEN_UD: - qp_init_attr.qp_type = IBV_QPT_UD; - break; - default: - IBCOM_ERR_CHKANDJUMP(1, -1, dprintf("invalid ibcom_open_flag\n")); - break; - } - qp_init_attr.sq_sig_all = 1; - - conp->icom_qp = ibv_create_qp(ib_pd, &qp_init_attr); - IBCOM_ERR_CHKANDJUMP(!conp->icom_qp, -1, printf("ibv_create_qp\n")); - - conp->max_send_wr = qp_init_attr.cap.max_send_wr; - conp->max_recv_wr = qp_init_attr.cap.max_recv_wr; - conp->max_inline_data = qp_init_attr.cap.max_inline_data; - - dprintf("ibcomOpen,max_send_wr=%d,max_recv_wr=%d,max_inline_data=%d\n", - qp_init_attr.cap.max_send_wr, qp_init_attr.cap.max_recv_wr, - qp_init_attr.cap.max_inline_data); - dprintf("ibcomOpen,fd=%d,qpn=%08x\n", *condesc, conp->icom_qp->qp_num); -#ifdef HAVE_LIBDCFA - dprintf("ibcomOpen,fd=%d,lid=%04x\n", *condesc, ib_ctx->lid); -#else - dprintf("ibcomOpen,fd=%d,lid=%04x\n", *condesc, conp->icom_pattr.lid); -#endif - -#ifdef HAVE_LIBDCFA - /* DCFA doesn't use gid */ - for (i = 0; i < 16; i++) { - conp->icom_gid.raw[i] = 0; - } -#else - ib_errno = ibv_query_gid(ib_ctx, ib_port, 0, &conp->icom_gid); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibv_query_gid\n")); - - dprintf("ibcomOpen,fd=%d,my_gid=", *condesc); - for (i = 0; i < 16; i++) { - dprintf("%02x", (int) conp->icom_gid.raw[i]); - } - dprintf("\n"); -#endif - - /* buffers */ - switch (ibcom_open_flag) { - case IBCOM_OPEN_RC: - /* RDMA-write-from and -to local memory area */ - conp->icom_mrlist = malloc(sizeof(struct ibv_mr *) * IBCOM_NBUF_RDMA); - memset(conp->icom_mrlist, 0, sizeof(struct ibv_mr *) * IBCOM_NBUF_RDMA); - conp->icom_mrlen = IBCOM_NBUF_RDMA; - conp->icom_mem = (void **) malloc(sizeof(void **) * IBCOM_NBUF_RDMA); - //printf("open,icom_mem=%p\n", conp->icom_mem); - memset(conp->icom_mem, 0, sizeof(void **) * IBCOM_NBUF_RDMA); - conp->icom_msize = (int *) malloc(sizeof(int *) * IBCOM_NBUF_RDMA); - memset(conp->icom_msize, 0, sizeof(int *) * IBCOM_NBUF_RDMA); - mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; - - /* RDMA-write-from local memory area */ - conp->icom_msize[IBCOM_RDMAWR_FROM] = IBCOM_RDMABUF_SZ; - conp->icom_mem[IBCOM_RDMAWR_FROM] = - mmap(0, IBCOM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("IbcomOpen,mmap=%p,len=%d\n", conp->icom_mem[IBCOM_RDMAWR_FROM], IBCOM_RDMABUF_SZ); - if (conp->icom_mem[IBCOM_RDMAWR_FROM] == (void *) -1) { - fprintf(stderr, "failed to allocate buffer\n"); - goto err_exit; - } - memset(conp->icom_mem[IBCOM_RDMAWR_FROM], 0, conp->icom_msize[IBCOM_RDMAWR_FROM]); - - conp->icom_mrlist[IBCOM_RDMAWR_FROM] = - ibcom_reg_mr_fetch(conp->icom_mem[IBCOM_RDMAWR_FROM], - conp->icom_msize[IBCOM_RDMAWR_FROM]); - if (!conp->icom_mrlist[IBCOM_RDMAWR_FROM]) { - fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); - goto err_exit; - } - - /* RDMA-write-to local memory area */ - conp->icom_msize[IBCOM_RDMAWR_TO] = IBCOM_RDMABUF_SZ; -#if 0 - int shmid = shmget(2, IBCOM_RDMABUF_SZ, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - IBCOM_ERR_CHKANDJUMP(shmid < 0, -1, perror("shmget")); - conp->icom_mem[IBCOM_RDMAWR_TO] = shmat(shmid, 0, 0); - if (conp->icom_mem[IBCOM_RDMAWR_TO] == (char *) -1) { - perror("Shared memory attach failure"); - shmctl(shmid, IPC_RMID, NULL); - ibcom_errno = -1; - goto fn_fail; - } -#else - conp->icom_mem[IBCOM_RDMAWR_TO] = - mmap(0, IBCOM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("IbcomOpen,mmap=%p,len=%d\n", conp->icom_mem[IBCOM_RDMAWR_TO], IBCOM_RDMABUF_SZ); -#endif - if (conp->icom_mem[IBCOM_RDMAWR_TO] == (void *) -1) { - fprintf(stderr, "failed to allocate buffer\n"); - goto err_exit; - } - memset(conp->icom_mem[IBCOM_RDMAWR_TO], 0, conp->icom_msize[IBCOM_RDMAWR_TO]); - - conp->icom_mrlist[IBCOM_RDMAWR_TO] = - ibv_reg_mr(ib_pd, conp->icom_mem[IBCOM_RDMAWR_TO], conp->icom_msize[IBCOM_RDMAWR_TO], - mr_flags); - if (!conp->icom_mrlist[IBCOM_RDMAWR_TO]) { - fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); - goto err_exit; - } -#ifdef HAVE_LIBDCFA - dprintf("ibcomOpen,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[IBCOM_RDMAWR_TO]->buf); -#else - dprintf("ibcomOpen,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[IBCOM_RDMAWR_TO]->addr); -#endif - dprintf("ibcomOpen,fd=%d,rkey=%08x\n", *condesc, conp->icom_mrlist[IBCOM_RDMAWR_TO]->rkey); - - /* RDMA-write-to remote memory area */ - conp->icom_rmem = (void **) malloc(sizeof(void **) * IBCOM_NBUF_RDMA); - if (conp->icom_rmem == 0) - goto err_exit; - memset(conp->icom_rmem, 0, sizeof(void **) * IBCOM_NBUF_RDMA); - - conp->icom_rsize = (size_t *) malloc(sizeof(void **) * IBCOM_NBUF_RDMA); - if (conp->icom_rsize == 0) - goto err_exit; - memset(conp->icom_rsize, 0, sizeof(void **) * IBCOM_NBUF_RDMA); - - conp->icom_rkey = (int *) malloc(sizeof(int) * IBCOM_NBUF_RDMA); - if (conp->icom_rkey == 0) - goto err_exit; - memset(conp->icom_rkey, 0, sizeof(int) * IBCOM_NBUF_RDMA); - break; - case IBCOM_OPEN_SCRATCH_PAD: - /* RDMA-write-from and -to local memory area */ - conp->icom_mrlist = malloc(sizeof(struct ibv_mr *) * IBCOM_NBUF_SCRATCH_PAD); - memset(conp->icom_mrlist, 0, sizeof(struct ibv_mr *) * IBCOM_NBUF_SCRATCH_PAD); - conp->icom_mrlen = IBCOM_NBUF_SCRATCH_PAD; - conp->icom_mem = (void **) malloc(sizeof(void **) * IBCOM_NBUF_SCRATCH_PAD); - memset(conp->icom_mem, 0, sizeof(void **) * IBCOM_NBUF_SCRATCH_PAD); - conp->icom_msize = (int *) malloc(sizeof(int *) * IBCOM_NBUF_SCRATCH_PAD); - memset(conp->icom_msize, 0, sizeof(int *) * IBCOM_NBUF_SCRATCH_PAD); - mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; - - /* RDMA-write-to remote memory area */ - conp->icom_rmem = (void **) malloc(sizeof(void **) * IBCOM_NBUF_SCRATCH_PAD); - IBCOM_ERR_CHKANDJUMP(conp->icom_rmem == 0, -1, dprintf("malloc failed\n")); - memset(conp->icom_rmem, 0, sizeof(void **) * IBCOM_NBUF_SCRATCH_PAD); - - conp->icom_rsize = (size_t *) malloc(sizeof(void **) * IBCOM_NBUF_SCRATCH_PAD); - IBCOM_ERR_CHKANDJUMP(conp->icom_rsize == 0, -1, dprintf("malloc failed\n")); - memset(conp->icom_rsize, 0, sizeof(void **) * IBCOM_NBUF_SCRATCH_PAD); - - conp->icom_rkey = (int *) malloc(sizeof(int) * IBCOM_NBUF_SCRATCH_PAD); - IBCOM_ERR_CHKANDJUMP(conp->icom_rkey == 0, -1, dprintf("malloc failed\n")); - memset(conp->icom_rkey, 0, sizeof(int) * IBCOM_NBUF_SCRATCH_PAD); - break; - - case IBCOM_OPEN_UD: - /* UD-write-from and -to local memory area */ - conp->icom_mrlist = malloc(sizeof(struct ibv_mr *) * IBCOM_NBUF_UD); - memset(conp->icom_mrlist, 0, sizeof(struct ibv_mr *) * IBCOM_NBUF_UD); - conp->icom_mrlen = IBCOM_NBUF_UD; - conp->icom_mem = (void **) malloc(sizeof(void **) * IBCOM_NBUF_UD); - memset(conp->icom_mem, 0, sizeof(void **) * IBCOM_NBUF_UD); - conp->icom_msize = (int *) malloc(sizeof(int *) * IBCOM_NBUF_UD); - memset(conp->icom_msize, 0, sizeof(int *) * IBCOM_NBUF_UD); - mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; - - /* UD-write-from local memory area */ - IBCOM_ERR_CHKANDJUMP(IBCOM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n")); - conp->icom_msize[IBCOM_UDWR_FROM] = IBCOM_UDBUF_SZ; - conp->icom_mem[IBCOM_UDWR_FROM] = - mmap(0, IBCOM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("IbcomOpen,mmap=%p,len=%d\n", conp->icom_mem[IBCOM_UDWR_FROM], IBCOM_UDBUF_SZ); - IBCOM_ERR_CHKANDJUMP(conp->icom_mem[IBCOM_UDWR_FROM] == (void *) -1, -1, - dprintf("failed to allocate buffer\n")); - memset(conp->icom_mem[IBCOM_UDWR_FROM], 0, conp->icom_msize[IBCOM_UDWR_FROM]); - - conp->icom_mrlist[IBCOM_UDWR_FROM] = - ibcom_reg_mr_fetch(conp->icom_mem[IBCOM_UDWR_FROM], conp->icom_msize[IBCOM_UDWR_FROM]); - IBCOM_ERR_CHKANDJUMP(!conp->icom_mrlist[IBCOM_UDWR_FROM], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); - - /* UD-write-to local memory area */ - /* addr to addr+39 are not filled, addr+40 to addr+length-1 are filled with payload */ - IBCOM_ERR_CHKANDJUMP(IBCOM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n")); - conp->icom_msize[IBCOM_UDWR_TO] = IBCOM_UDBUF_SZ; - conp->icom_mem[IBCOM_UDWR_TO] = - mmap(0, IBCOM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("IbcomOpen,mmap=%p,len=%d\n", conp->icom_mem[IBCOM_UDWR_TO], IBCOM_UDBUF_SZ); - IBCOM_ERR_CHKANDJUMP(conp->icom_mem[IBCOM_UDWR_TO] == (void *) -1, -1, - dprintf("failed to allocate buffer\n")); - memset(conp->icom_mem[IBCOM_UDWR_TO], 0, conp->icom_msize[IBCOM_UDWR_TO]); - - conp->icom_mrlist[IBCOM_UDWR_TO] = - ibcom_reg_mr_fetch(conp->icom_mem[IBCOM_UDWR_TO], conp->icom_msize[IBCOM_UDWR_TO]); - IBCOM_ERR_CHKANDJUMP(!conp->icom_mrlist[IBCOM_UDWR_TO], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); - - /* initialize arena allocator for IBCOM_UDWR_TO */ - //ibcom_udbuf_init(conp->icom_mem[IBCOM_UDWR_TO]); - - dprintf("ibcomOpen,ud,fd=%d,lkey=%08x\n", *condesc, conp->icom_mrlist[IBCOM_UDWR_TO]->lkey); - break; - default: - IBCOM_ERR_CHKANDJUMP(1, -1, dprintf("invalid ibcom_open_flag\n")); - break; - - } - - /* command templates */ - switch (ibcom_open_flag) { - case IBCOM_OPEN_RC: - - /* SR (send request) template */ - conp->icom_sr = - (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * IBCOM_RC_SR_NTEMPLATE); - memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * IBCOM_RC_SR_NTEMPLATE); - - for (i = 0; i < IBCOM_SMT_INLINE_NCHAIN; i++) { - /* SGE (RDMA-send-from memory) template */ -#ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[0]), 0, - sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = - (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_SMT_INLINE_INITIATOR_NSGE); - memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SMT_INLINE_INITIATOR_NSGE); -#endif - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].next = - (i == - IBCOM_SMT_INLINE_NCHAIN - 1) ? NULL : &conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + - i + 1]; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list = sge; -#endif - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].opcode = IBV_WR_RDMA_WRITE; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].send_flags = - IBV_SEND_SIGNALED | IBV_SEND_INLINE; - } - - { -#ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[0]), 0, - sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = - (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * - IBCOM_SMT_NOINLINE_INITIATOR_NSGE); - memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SMT_NOINLINE_INITIATOR_NSGE); -#endif - conp->icom_sr[IBCOM_SMT_NOINLINE].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list = sge; -#endif - conp->icom_sr[IBCOM_SMT_NOINLINE].opcode = IBV_WR_RDMA_WRITE; - conp->icom_sr[IBCOM_SMT_NOINLINE].send_flags = IBV_SEND_SIGNALED; - } - { - /* SR (send request) template for IBCOM_LMT_INITIATOR */ -#ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[0]), 0, - sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = - (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_LMT_INITIATOR_NSGE); - memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_INITIATOR_NSGE); -#endif - conp->icom_sr[IBCOM_LMT_INITIATOR].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list = sge; -#endif - conp->icom_sr[IBCOM_LMT_INITIATOR].opcode = IBV_WR_RDMA_READ; - conp->icom_sr[IBCOM_LMT_INITIATOR].send_flags = IBV_SEND_SIGNALED; - } - - /* SR (send request) template for IBCOM_LMT_PUT *//* for lmt-put-done */ -#ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[IBCOM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE); - memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE); -#endif - conp->icom_sr[IBCOM_LMT_PUT].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_LMT_PUT].sg_list = sge; -#endif - conp->icom_sr[IBCOM_LMT_PUT].opcode = IBV_WR_RDMA_WRITE; - conp->icom_sr[IBCOM_LMT_PUT].send_flags = IBV_SEND_SIGNALED; - - /* SR (send request) template for IBCOM_RDMAWR_FRMFIXED */ - /* not implemented */ - - /* SGE (scatter gather element) template for recv */ - /* nothing is required for RDMA-write */ - - /* RR (receive request) template for IBCOM_RDMAWR_RESPONDER */ - conp->icom_rr = - (struct ibv_recv_wr *) malloc(sizeof(struct ibv_recv_wr) * IBCOM_RC_RR_NTEMPLATE); - memset(conp->icom_rr, 0, sizeof(struct ibv_recv_wr) * IBCOM_RC_RR_NTEMPLATE); - - /* create one dummy RR to ibv_post_recv */ - conp->icom_rr[IBCOM_RDMAWR_RESPONDER].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_rr[IBCOM_RDMAWR_RESPONDER].sg_list = NULL; -#endif - conp->icom_rr[IBCOM_RDMAWR_RESPONDER].num_sge = 0; - break; - - case IBCOM_OPEN_SCRATCH_PAD:{ - /* SR (send request) template */ - conp->icom_sr = - (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * - IBCOM_SCRATCH_PAD_SR_NTEMPLATE); - memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * IBCOM_SCRATCH_PAD_SR_NTEMPLATE); - - /* SR (send request) template for IBCOM_SCRATCH_PAD_INITIATOR */ -#ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0]), 0, - sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = - (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * - IBCOM_SCRATCH_PAD_INITIATOR_NSGE); - memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SCRATCH_PAD_INITIATOR_NSGE); -#endif - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list = sge; -#endif - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].num_sge = 1; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].opcode = IBV_WR_RDMA_WRITE; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].send_flags = IBV_SEND_SIGNALED; - break; - } - - case IBCOM_OPEN_RC_LMT_PUT: - /* SR (send request) template */ - conp->icom_sr = - (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * - IBCOM_RC_SR_LMT_PUT_NTEMPLATE); - memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * IBCOM_RC_SR_LMT_PUT_NTEMPLATE); - /* SR (send request) template for IBCOM_LMT_PUT */ -#ifdef HAVE_LIBDCFA - memset(&(conp->icom_sr[IBCOM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE); - memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_PUT_NSGE); -#endif - conp->icom_sr[IBCOM_LMT_PUT].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_LMT_PUT].sg_list = sge; -#endif - conp->icom_sr[IBCOM_LMT_PUT].opcode = IBV_WR_RDMA_WRITE; - conp->icom_sr[IBCOM_LMT_PUT].send_flags = IBV_SEND_SIGNALED; - break; - - case IBCOM_OPEN_UD: - /* SGE (RDMA-send-from memory) template for IBCOM_UD_INITIATOR */ -#ifdef HAVE_LIBDCFA - sge = &(conp->icom_sr[IBCOM_UD_INITIATOR].sg_list[0]); - memset(sge, 0, sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = (struct ibv_sge *) calloc(1, sizeof(struct ibv_sge)); -#endif - /* addr to addr + length - 1 will be on the payload, but search backword for "<= 40" */ - sge[0].addr = (uint64_t) conp->icom_mem[IBCOM_UDWR_FROM] + 40; - sge[0].length = IBCOM_UDBUF_SZSEG - 40; - sge[0].lkey = conp->icom_mrlist[IBCOM_UDWR_FROM]->lkey; - - - conp->icom_ah_attr = - (struct ibv_ah_attr *) calloc(IBCOM_UD_SR_NTEMPLATE, sizeof(struct ibv_ah_attr)); - - conp->icom_ah_attr[IBCOM_UD_INITIATOR].sl = 0; - conp->icom_ah_attr[IBCOM_UD_INITIATOR].src_path_bits = 0; - conp->icom_ah_attr[IBCOM_UD_INITIATOR].static_rate = 0; /* not limit on static rate (100% port speed) */ - conp->icom_ah_attr[IBCOM_UD_INITIATOR].is_global = 0; - conp->icom_ah_attr[IBCOM_UD_INITIATOR].port_num = conp->icom_port; - -#if 0 - conp->icom_ah_attr[IBCOM_UD_INITIATOR].is_global = 1; - conp->icom_ah_attr[IBCOM_UD_INITIATOR].grh.flow_label = 0; - conp->icom_ah_attr[IBCOM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */ - conp->icom_ah_attr[IBCOM_UD_INITIATOR].grh.hop_limit = 1; - conp->icom_ah_attr[IBCOM_UD_INITIATOR].grh.traffic_class = 0; -#endif - - /* SR (send request) template for IBCOM_UD_INITIATOR */ - conp->icom_sr = - (struct ibv_send_wr *) calloc(IBCOM_UD_SR_NTEMPLATE, sizeof(struct ibv_send_wr)); - - conp->icom_sr[IBCOM_UD_INITIATOR].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_UD_INITIATOR].sg_list = sge; -#endif - conp->icom_sr[IBCOM_UD_INITIATOR].num_sge = 1; - conp->icom_sr[IBCOM_UD_INITIATOR].opcode = IBV_WR_SEND; - conp->icom_sr[IBCOM_UD_INITIATOR].send_flags = IBV_SEND_SIGNALED; - - conp->icom_sr[IBCOM_UD_INITIATOR].wr.ud.remote_qkey = IBCOM_QKEY; - - /* SGE (scatter gather element) template for recv */ -#ifdef HAVE_LIBDCFA - sge = &(conp->icom_rr[IBCOM_UD_RESPONDER].sg_list[0]); - memset(sge, 0, sizeof(struct ibv_sge) * WR_SG_NUM); -#else - sge = (struct ibv_sge *) calloc(1, sizeof(struct ibv_sge)); -#endif - sge[0].addr = (uint64_t) conp->icom_mem[IBCOM_UDWR_TO]; - sge[0].length = IBCOM_UDBUF_SZ; - sge[0].lkey = conp->icom_mrlist[IBCOM_UDWR_TO]->lkey; - - /* RR (receive request) template for IBCOM_UD_RESPONDER */ - conp->icom_rr = - (struct ibv_recv_wr *) calloc(IBCOM_UD_RR_NTEMPLATE, sizeof(struct ibv_recv_wr)); - - /* create one dummy RR to ibv_post_recv */ - conp->icom_rr[IBCOM_UD_RESPONDER].next = NULL; -#ifdef HAVE_LIBDCFA -#else - conp->icom_rr[IBCOM_UD_RESPONDER].sg_list = sge; -#endif - conp->icom_rr[IBCOM_UD_RESPONDER].num_sge = 1; - break; - } - - maxcon++; - - fn_exit: - return ibcom_errno; - err_exit: - ibcomClean(conp); - return -1; - fn_fail: - ibcomClean(conp); - goto fn_exit; -} - -/* 1. allocate memory area if it's not allocated or reuse it if it's allocated - 2. ibv_reg_mr it and store rkey to conp->icom_mrlist - buf is output */ -int ibcom_alloc(int condesc, int sz) -{ - IbCom *conp; - int ibcom_errno = 0; - int mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - - switch (conp->open_flag) { - - case IBCOM_OPEN_SCRATCH_PAD: - /* RDMA-write-to local memory area */ - if (!scratch_pad) { - scratch_pad = mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - dprintf("ibcom_alloc,mmap=%p,len=%d\n", scratch_pad, sz); - IBCOM_ERR_CHKANDJUMP(scratch_pad == (void *) -1, -1, - dprintf("failed to allocate buffer\n")); - dprintf("ibcom_alloc,scratch_pad=%p\n", scratch_pad); - memset(scratch_pad, 0, sz); - } - conp->icom_mem[IBCOM_SCRATCH_PAD_TO] = scratch_pad; - conp->icom_msize[IBCOM_SCRATCH_PAD_TO] = sz; - - conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO] = - ibcom_reg_mr_fetch(conp->icom_mem[IBCOM_SCRATCH_PAD_TO], - conp->icom_msize[IBCOM_SCRATCH_PAD_TO]); - IBCOM_ERR_CHKANDJUMP(!conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO], -1, - dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); - -#ifdef HAVE_LIBDCFA - dprintf("ibcom_alloc,fd=%d,rmem=%p\n", condesc, - conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO]->buf); -#else - dprintf("ibcom_alloc,fd=%d,rmem=%p\n", condesc, - conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO]->addr); -#endif - dprintf("ibcom_alloc,fd=%d,rkey=%08x\n", condesc, - conp->icom_mrlist[IBCOM_SCRATCH_PAD_TO]->rkey); - break; - default: - IBCOM_ERR_CHKANDJUMP(1, -1, - dprintf("ibcom_alloc, invalid open_flag=%d\n", conp->open_flag)); - break; - } - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_close(int condesc) -{ - IbCom *conp; - int ibcom_errno = 0; - - dprintf("ibcom_close,condesc=%d\n", condesc); - - RANGE_CHECK_WITH_ERROR(condesc, conp); - ibcomClean(conp); - --maxcon; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid) -{ - IbCom *conp; - int ibcom_errno = 0; - int ib_errno; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - if (conp->icom_connected == 1) { - ibcom_errno = -1; - goto fn_fail; - } - - struct ibv_qp_attr attr; - int flags; - - switch (conp->open_flag) { - case IBCOM_OPEN_RC: - case IBCOM_OPEN_RC_LMT_PUT: - case IBCOM_OPEN_SCRATCH_PAD: - /* Init QP */ - ib_errno = modify_qp_to_init(conp->icom_qp, conp->icom_port); - if (ib_errno) { - fprintf(stderr, "change QP state to INIT failed\n"); - ibcom_errno = ib_errno; - goto fn_fail; - } - /* Modify QP TO RTR status */ - ib_errno = - modify_qp_to_rtr(conp->icom_qp, remote_qpnum, remote_lid, remote_gid, conp->icom_port, - 0); - conp->remote_lid = remote_lid; /* for debug */ - if (ib_errno) { - fprintf(stderr, "failed to modify QP state to RTR\n"); - ibcom_errno = ib_errno; - goto fn_fail; - } - /* Modify QP TO RTS status */ - ib_errno = modify_qp_to_rts(conp->icom_qp); - if (ib_errno) { - fprintf(stderr, "failed to modify QP state to RTS\n"); - ibcom_errno = ib_errno; - goto fn_fail; - } - break; - case IBCOM_OPEN_UD: - /* INIT */ - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_INIT; - attr.port_num = conp->icom_port; - attr.pkey_index = 0; - attr.qkey = IBCOM_QKEY; - flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY; - ib_errno = ibv_modify_qp(conp->icom_qp, &attr, flags); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_modify_qp")); - - /* RTR */ - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RTR; - flags = IBV_QP_STATE; - ib_errno = ibv_modify_qp(conp->icom_qp, &attr, flags); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_modify_qp")); - - /* RTS */ - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RTS; - attr.sq_psn = 0; - flags = IBV_QP_STATE | IBV_QP_SQ_PSN; - ib_errno = ibv_modify_qp(conp->icom_qp, &attr, flags); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_modify_qp")); - break; - } - conp->icom_connected = 1; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -#define MPID_NEM_DCFA_ENABLE_INLINE -int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, int sz_hdr, - void *data, int sz_data, int *copied) -{ - IbCom *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - int num_sge; - - dprintf("ibcom_isend,prefix=%p,sz_prefix=%d,hdr=%p,sz_hdr=%d,data=%p,sz_data=%d\n", prefix, - sz_prefix, hdr, sz_hdr, data, sz_data); - - RANGE_CHECK_WITH_ERROR(condesc, conp); - if (conp->icom_connected == 0) { - return -1; - } - int sz_data_pow2; - DCFA_NEM_SZ_DATA_POW2(sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data); - uint32_t sumsz = sz_data_pow2 + sizeof(tailmagic_t); - if (sz_data > 16000) { - //dprintf("ibcom_isend,sz_data=%d,sz_data_pow2=%d,sz_max=%ld\n", sz_data, sz_data_pow2, DCFA_NEM_MAX_DATA_POW2); - } - - num_sge = 0; - - void *buf_from = - (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_FROM] + - IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); - - sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf_from; - sz_hdrmagic->sz = sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data + sizeof(tailmagic_t); - sz_hdrmagic->magic = IBCOM_MAGIC; - - /* memcpy hdr is needed because hdr resides in stack when sending close-VC command */ - /* memcpy is performed onto IBCOM_RDMAWR_FROM buffer */ - void *hdr_copy = (uint8_t *) buf_from + sizeof(sz_hdrmagic_t); - memcpy(hdr_copy, prefix, sz_prefix); - memcpy((uint8_t *) hdr_copy + sz_prefix, hdr, sz_hdr); -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) sz_hdrmagic; - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->host_addr + ((uint64_t) sz_hdrmagic - - (uint64_t) conp-> - icom_mem[IBCOM_RDMAWR_FROM]); -#else - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) sz_hdrmagic; -#endif - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].length = - sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr; - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].lkey = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->lkey; - num_sge += 1; - - if (sz_data) { - //dprintf("ibcom_isend,data=%p,sz_data=%d\n", data, sz_data); - struct ibv_mr *mr_data = ibcom_reg_mr_fetch(data, sz_data); - IBCOM_ERR_CHKANDJUMP(!mr_data, -1, printf("ibcom_isend,ibv_reg_mr_fetch failed\n")); -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) data; - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = - mr_data->host_addr + ((uint64_t) data - (uint64_t) data); -#else - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) data; -#endif - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].length = sz_data; - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].lkey = mr_data->lkey; - num_sge += 1; - } - - int sz_pad = sz_data_pow2 - (sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data); - tailmagic_t *tailmagic = - (tailmagic_t *) ((uint8_t *) buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad); - tailmagic->magic = IBCOM_MAGIC; -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = - (uint64_t) buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr; - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from + - sizeof(sz_hdrmagic_t) + sz_prefix + - sz_hdr - - (uint64_t) conp-> - icom_mem[IBCOM_RDMAWR_FROM]); -#else - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = - (uint64_t) buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr; -#endif - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].length = sz_pad + sizeof(tailmagic_t); - conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].lkey = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->lkey; - num_sge += 1; - dprintf("ibcom_isend,sz_data=%d,pow2=%d,sz_pad=%d,num_sge=%d\n", sz_data, sz_data_pow2, sz_pad, - num_sge); - - conp->icom_sr[IBCOM_SMT_NOINLINE].num_sge = num_sge; - conp->icom_sr[IBCOM_SMT_NOINLINE].wr_id = wr_id; - conp->icom_sr[IBCOM_SMT_NOINLINE].wr.rdma.remote_addr = - (uint64_t) conp->icom_rmem[IBCOM_RDMAWR_TO] + - IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); - /* rkey is defined in ibcom_reg_mr_connect */ - - //dprintf("ibcom_isend,condesc=%d,num_sge=%d,opcode=%08x,imm_data=%08x,wr_id=%016lx, raddr=%p, rkey=%08x\n", condesc, conp->icom_sr[IBCOM_SMT_NOINLINE].num_sge, conp->icom_sr[IBCOM_SMT_NOINLINE].opcode, conp->icom_sr[IBCOM_SMT_NOINLINE].imm_data, conp->icom_sr[IBCOM_SMT_NOINLINE].wr_id, conp->icom_sr[IBCOM_SMT_NOINLINE].wr.rdma.remote_addr, conp->icom_sr[IBCOM_SMT_NOINLINE].wr.rdma.rkey); - - /* other commands can executed RDMA-rd command */ - /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ -#if 0 - if (conp->after_rdma_rd) { - conp->icom_sr[IBCOM_SMT_NOINLINE].send_flags |= IBV_SEND_FENCE; - } -#endif -#ifdef MPID_NEM_DCFA_ENABLE_INLINE - if (sumsz <= conp->max_inline_data) { - conp->icom_sr[IBCOM_SMT_NOINLINE].send_flags |= IBV_SEND_INLINE; - *copied = 1; - } - else { - *copied = 0; - } -#endif -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_NOINLINE]); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_isend, ibv_post_send, rc=%d\n", ib_errno)); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_NOINLINE], &bad_wr); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); -#endif -#ifdef MPID_NEM_DCFA_ENABLE_INLINE - if (sumsz <= conp->max_inline_data) { - conp->icom_sr[IBCOM_SMT_NOINLINE].send_flags &= ~IBV_SEND_INLINE; - } -#endif -#if 0 - if (conp->after_rdma_rd) { - conp->icom_sr[IBCOM_SMT_NOINLINE].send_flags &= ~IBV_SEND_FENCE; - conp->after_rdma_rd = 0; - } -#endif - - conp->sseq_num += 1; - assert(conp->sseq_num > 0); - conp->ncom += 1; - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, int sz_data) -{ - IbCom *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - int sz_data_rem = sz_data; - int i; - struct ibv_mr *mr_data; - uint32_t sumsz = sizeof(sz_hdrmagic_t) + sz_hdr + sz_data + sizeof(tailmagic_t); - unsigned long tscs, tsce; - - dprintf("ibcom_isend_chain,enter\n"); - RANGE_CHECK_WITH_ERROR(condesc, conp); - IBCOM_ERR_CHKANDJUMP(conp->icom_connected == 0, -1, - printf("ibcom_isend_chain,icom_connected==0\n")); - - void *buf_from = - (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_FROM] + - IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); - - /* make a tail-magic position is in a fixed set */ - int sz_data_pow2; - DCFA_NEM_SZ_DATA_POW2(sizeof(sz_hdrmagic_t) + sz_hdr + sz_data); - - /* let the last command icom_sr[IBCOM_SMT_INLINE_CHAIN-1] which has IBV_WR_RDMA_WRITE_WITH_IMM */ - int s = - IBCOM_SMT_INLINE_NCHAIN - (sizeof(sz_hdrmagic_t) + sz_hdr + sz_data_pow2 + - sizeof(tailmagic_t) + IBCOM_INLINE_DATA - 1) / IBCOM_INLINE_DATA; - IBCOM_ERR_CHKANDJUMP((sizeof(sz_hdrmagic_t) + sz_hdr + sz_data_pow2) % 4 != 0, -1, - printf("ibcom_isend_chain,tail-magic gets over packet-boundary\n")); - IBCOM_ERR_CHKANDJUMP(s < 0 || - s >= IBCOM_SMT_INLINE_NCHAIN, -1, printf("ibcom_isend_chain,s\n")); - dprintf("ibcom_isend_chain,sz_hdr=%d,sz_data=%d,s=%d\n", sz_hdr, sz_data, s); - - for (i = s; i < IBCOM_SMT_INLINE_NCHAIN; i++) { - - //tscs = MPID_nem_dcfa_rdtsc(); - int sz_used = 0; /* how much of the payload of a IB packet is used? */ - int num_sge = 0; - if (i == s) { - sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf_from; - sz_hdrmagic->sz = sumsz; - sz_hdrmagic->magic = IBCOM_MAGIC; - memcpy((uint8_t *) buf_from + sizeof(sz_hdrmagic_t), hdr, sz_hdr); -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) buf_from; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) conp-> - icom_mem[IBCOM_RDMAWR_FROM]); -#else - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) buf_from; -#endif - buf_from = (uint8_t *) buf_from + sizeof(sz_hdrmagic_t) + sz_hdr; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = - sizeof(sz_hdrmagic_t) + sz_hdr; - sz_used += sizeof(sz_hdrmagic_t) + sz_hdr; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->lkey; - num_sge += 1; - dprintf("ibcom_isend_chain,i=%d,sz_used=%d\n", i, sz_used); - } - //tsce = MPID_nem_dcfa_rdtsc(); printf("0,%ld\n", tsce-tscs); - - //tscs = MPID_nem_dcfa_rdtsc(); - if (sz_data_rem > 0) { -#ifdef HAVE_LIBDCFA -#else - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) data + sz_data - sz_data_rem; -#endif - int sz_data_red = - sz_used + sz_data_rem + sizeof(tailmagic_t) <= - IBCOM_INLINE_DATA ? sz_data_rem : sz_data_rem <= - IBCOM_INLINE_DATA - sz_used ? sz_data_rem : IBCOM_INLINE_DATA - sz_used; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_data_red; - sz_used += sz_data_red; - sz_data_rem -= sz_data_red; - IBCOM_ERR_CHKANDJUMP(sz_data_rem < 0, -1, printf("ibcom_isend_chain,sz_data_rem\n")); - - if (i == s) { - IBCOM_ERR_CHKANDJUMP(!sz_data, -1, printf("ibcom_isend_chain,sz_data==0\n")); - mr_data = ibcom_reg_mr_fetch(data, sz_data); - IBCOM_ERR_CHKANDJUMP(!mr_data, -1, printf("ibcom_isend,ibv_reg_mr_fetch failed\n")); - } -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) data + sz_data - sz_data_rem; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - mr_data->host_addr + ((uint64_t) data + sz_data - sz_data_rem - (uint64_t) data); -#endif - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = mr_data->lkey; - num_sge += 1; - dprintf("ibcom_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used, sz_data_rem); - } - else { /* tailmagic only packet is being generated */ - - } - //tsce = MPID_nem_dcfa_rdtsc(); printf("1,%ld\n", tsce-tscs); - - //tscs = MPID_nem_dcfa_rdtsc(); - if (i == IBCOM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */ - int sz_pad = sz_data_pow2 - sz_data; - tailmagic_t *tailmagic = (tailmagic_t *) ((uint8_t *) buf_from + sz_pad); - tailmagic->magic = IBCOM_MAGIC; -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) buf_from; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) conp-> - icom_mem[IBCOM_RDMAWR_FROM]); -#else - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) buf_from; -#endif - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = - sz_pad + sizeof(tailmagic_t); - sz_used += sz_pad + sizeof(tailmagic_t); - IBCOM_ERR_CHKANDJUMP(sz_data_rem != 0, -1, printf("ibcom_isend_chain, sz_data_rem\n")); - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->lkey; - num_sge += 1; - - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].imm_data = conp->sseq_num; - dprintf("ibcom_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad, sz_used, - num_sge); - } - else if (IBCOM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */ - IBCOM_ERR_CHKANDJUMP(1, -1, - printf - ("ibcom_isend_chain,tail-magic gets over packet-boundary\n")); - int sz_pad = IBCOM_INLINE_DATA - sz_used; -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = - (uint64_t) buf_from; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - - (uint64_t) conp-> - icom_mem[IBCOM_RDMAWR_FROM]); -#else - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = - (uint64_t) buf_from; -#endif - buf_from = (uint8_t *) buf_from + sz_pad; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad; - sz_used += sz_pad; - IBCOM_ERR_CHKANDJUMP(sz_used != IBCOM_INLINE_DATA, -1, - printf("ibcom_isend_chain, sz_used\n")); - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = - conp->icom_mrlist[IBCOM_RDMAWR_FROM]->lkey; - num_sge += 1; - dprintf("ibcom_isend_chain,i=%d,sz_pad=%d,sz_used=%d\n", i, sz_pad, sz_used); - } - else { /* packet is full with data */ - IBCOM_ERR_CHKANDJUMP(sz_used != IBCOM_INLINE_DATA, -1, - printf("ibcom_isend_chain, sz_used\n")); - } - //tsce = MPID_nem_dcfa_rdtsc(); printf("2,%ld\n", tsce-tscs); - - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].num_sge = num_sge; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].wr_id = wr_id; - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].wr.rdma.remote_addr = - (uint64_t) conp->icom_rmem[IBCOM_RDMAWR_TO] + - IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG) + IBCOM_INLINE_DATA * (i - - s); - } -#if 0 - if (conp->after_rdma_rd) { - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s].send_flags |= IBV_SEND_FENCE; - } -#endif -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s]); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s], &bad_wr); -#endif -#if 0 - if (i == 0 && conp->after_rdma_rd) { - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + s].send_flags &= ~IBV_SEND_FENCE; - conp->after_rdma_rd = 0; - } -#endif -#ifdef HAVE_LIBDCFA - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_isend, ibv_post_send, rc=%d\n", ib_errno)); -#else - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); -#endif - conp->ncom += (IBCOM_SMT_INLINE_NCHAIN - s); - conp->sseq_num += 1; - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_irecv(int condesc, uint64_t wr_id) -{ - - IbCom *conp; - int ib_errno; - int ibcom_errno = 0; - struct ibv_recv_wr *bad_wr; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - // if (conp->icom_connected == 0) { return -1; } - - //dprintf("ibcom_irecv,condesc=%d,wr_id=%016lx\n", condesc, wr_id); - - conp->icom_rr[IBCOM_RDMAWR_RESPONDER].wr_id = wr_id; -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_RDMAWR_RESPONDER]); -#else - ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_RDMAWR_RESPONDER], &bad_wr); -#endif - if (ib_errno) { -#ifdef HAVE_LIBDCFA - fprintf(stderr, "ibcom_irecv: failed to post receive, ib_errno=%d\n", ib_errno); -#else - fprintf(stderr, "ibcom_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n", ib_errno, - bad_wr); -#endif - ibcom_errno = ib_errno; - goto fn_fail; - } - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, uint32_t remote_qpn, - uint32_t imm_data, uint64_t wr_id) -{ - IbCom *conp; - struct ibv_send_wr *bad_wr; - int ibcom_errno = 0, ib_errno; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - -#ifdef HAVE_LIBDCFA - IBCOM_ERR_CHKANDJUMP(1, -1, - dprintf - ("ibcom_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n")); -#else - /* prepare ibv_ah_attr */ - conp->icom_ah_attr[IBCOM_UD_INITIATOR].dlid = remote_lid; -#if 0 - conp->icom_ah_attr[IBCOM_UD_INITIATOR].grh.dgid = *remote_gid; -#endif - - /* prepare ibv_ah */ - struct ibv_ah *ah; - ah = ibv_create_ah(ib_pd, &conp->icom_ah_attr[IBCOM_UD_INITIATOR]); - IBCOM_ERR_CHKANDJUMP(!ah, -1, dprintf("ibv_crate_ah\n")); - - conp->icom_sr[IBCOM_UD_INITIATOR].wr.ud.ah = ah; - conp->icom_sr[IBCOM_UD_INITIATOR].wr.ud.remote_qpn = remote_qpn; - /* qkey is defined in open */ - - //dprintf("lid=%04x\n", conp->icom_ah_attr[IBCOM_UD_INITIATOR].dlid); - //dprintf("qpn=%08x\n", conp->icom_sr[IBCOM_UD_INITIATOR].wr.ud.remote_qpn); - - /* recv doesn't know the length, so we can't optimize it */ - // conp->icom_sr[IBCOM_UD_INITIATOR].sg_list[0].length = length; - - conp->icom_sr[IBCOM_UD_INITIATOR].wr_id = wr_id; - conp->icom_sr[IBCOM_UD_INITIATOR].imm_data = imm_data; - -#if 0 - if (length <= qpinfo->max_inline_data) { - conp->icom_sr[IBCOM_UD_INITIATOR].send_flags |= IBV_SEND_INLINE; - } -#endif - -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_UD_INITIATOR]); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_UD_INITIATOR], &bad_wr); -#endif - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_post_send")); -#endif /* DCFA */ - - conp->ncom += 1; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_udrecv(int condesc) -{ - IbCom *conp; - struct ibv_recv_wr *bad_wr; - int ibcom_errno = 0, ib_errno; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - - /* Create RR */ - conp->icom_rr[IBCOM_UD_RESPONDER].wr_id = 0; - - /* Post RR to RQ */ -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_UD_RESPONDER]); -#else - ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[IBCOM_UD_RESPONDER], &bad_wr); -#endif - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibv_post_recv ib_errno=%d\n", ib_errno)); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr) -{ - IbCom *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - int num_sge; - - dprintf("ibcom_lrecv,enter,raddr=%p,sz_data=%d,laddr=%p\n", raddr, sz_data, laddr); - - RANGE_CHECK_WITH_ERROR(condesc, conp); - IBCOM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("ibcom_lrecv,not connected\n")); - IBCOM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("ibcom_lrecv,sz_data==0\n")); - - num_sge = 0; - - /* register memory area containing data */ - struct ibv_mr *mr_data = ibcom_reg_mr_fetch(laddr, sz_data); - IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_lrecv,ibv_reg_mr_fetch failed\n")); - - /* Erase magic, super bug!! */ - //((tailmagic_t*)(laddr + sz_data - sizeof(tailmagic_t)))->magic = 0; -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].mic_addr = (uint64_t) laddr; - conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].addr = - mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); -#else - conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].addr = (uint64_t) laddr; -#endif - conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].length = sz_data; - conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[num_sge].lkey = mr_data->lkey; - num_sge += 1; - - conp->icom_sr[IBCOM_LMT_INITIATOR].num_sge = num_sge; - conp->icom_sr[IBCOM_LMT_INITIATOR].wr_id = wr_id; - conp->icom_sr[IBCOM_LMT_INITIATOR].wr.rdma.remote_addr = (uint64_t) raddr; - conp->icom_sr[IBCOM_LMT_INITIATOR].wr.rdma.rkey = rkey; - -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_LMT_INITIATOR]); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_lrecv, ibv_post_send, rc=%d\n", ib_errno)); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_LMT_INITIATOR], &bad_wr); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_lrecv, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); -#endif - - /* other commands can be executed before RDMA-rd command */ - /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ -#if 0 - conp->after_rdma_rd = 1; -#endif - conp->ncom += 1; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -/* use the same QP as isend */ -int ibcom_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr) -{ - IbCom *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - int num_sge; - - dprintf("ibcom_put_lmt,enter,sz_data=%d,laddr=%p\n", sz_data, laddr); - - RANGE_CHECK_WITH_ERROR(condesc, conp); - IBCOM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("ibcom_put_lmt,not connected\n")); - IBCOM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("ibcom_put_lmt,sz_data==0\n")); - - num_sge = 0; - - /* register memory area containing data */ - struct ibv_mr *mr_data = ibcom_reg_mr_fetch(laddr, sz_data); - IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_put_lmt,ibv_reg_mr_fetch failed\n")); - -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].mic_addr = (uint64_t) laddr; - conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].addr = - mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); -#else - conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].addr = (uint64_t) laddr; -#endif - conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].length = sz_data; - conp->icom_sr[IBCOM_LMT_PUT].sg_list[num_sge].lkey = mr_data->lkey; - num_sge += 1; - - conp->icom_sr[IBCOM_LMT_PUT].num_sge = num_sge; - conp->icom_sr[IBCOM_LMT_PUT].wr_id = wr_id; - conp->icom_sr[IBCOM_LMT_PUT].wr.rdma.remote_addr = (uint64_t) raddr; - conp->icom_sr[IBCOM_LMT_PUT].wr.rdma.rkey = rkey; - -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_LMT_PUT]); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibcom_put_lmt, ibv_post_send, rc=%d\n", ib_errno)); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_LMT_PUT], &bad_wr); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_put_lmt, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, - bad_wr)); -#endif - - conp->ncom += 1; - dprintf("ibcom_put_lmt,exit\n"); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr) -{ - IbCom *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - - dprintf("ibcom_put_scratch_pad,enter,wr_id=%llx,offset=%llx,sz=%d,laddr=%p\n", - (unsigned long long) wr_id, (unsigned long long) offset, sz, laddr); - dprintf("ibcom_put_scratch_pad,data=%08x\n", *((uint32_t *) laddr)); - - RANGE_CHECK_WITH_ERROR(condesc, conp); - IBCOM_ERR_CHKANDJUMP(conp->open_flag != IBCOM_OPEN_SCRATCH_PAD, -1, - dprintf("ibcom_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag)); - IBCOM_ERR_CHKANDJUMP(!conp->icom_connected, -1, - dprintf("ibcom_put_scratch_pad,not connected\n")); - IBCOM_ERR_CHKANDJUMP(!sz, -1, dprintf("ibcom_put_scratch_pad,sz==0\n")); - - /* register memory area containing data */ - struct ibv_mr *mr_data = ibcom_reg_mr_fetch(laddr, sz); - IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_put_scratch_pad,ibv_reg_mr_fetch failed\n")); - dprintf("ibcom_put_scratch_pad,"); - -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].mic_addr = (uint64_t) laddr; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = - mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); -#else - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = (uint64_t) laddr; -#endif - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].length = sz; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].lkey = mr_data->lkey; - - /* num_sge is defined in ibcomOpen */ - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr_id = wr_id; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr = - (uint64_t) conp->icom_rmem[IBCOM_SCRATCH_PAD_TO] + offset; - /* rkey is defined in ibcom_reg_mr_connect */ - - dprintf("ibcom_put_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr); - -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR]); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno)); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR], &bad_wr); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", - ib_errno, bad_wr)); -#endif - - conp->ncom_scratch_pad += 1; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -#ifdef DCFA_ONDEMAND -int ibcom_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, uint64_t compare, - uint64_t swap) -{ - IbCom *conp; - int ibcom_errno = 0; - struct ibv_send_wr *bad_wr; - int ib_errno; - - dprintf("ibcom_put_scratch_pad,enter,wr_id=%llx,offset=%llx,sz=%d,laddr=%p\n", - (unsigned long long) wr_id, (unsigned long long) offset, sz, laddr); - dprintf("ibcom_put_scratch_pad,data=%08x\n", *((uint32_t *) laddr)); - - RANGE_CHECK_WITH_ERROR(condesc, conp); - IBCOM_ERR_CHKANDJUMP(conp->open_flag != IBCOM_OPEN_SCRATCH_PAD, -1, - dprintf("ibcom_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag)); - IBCOM_ERR_CHKANDJUMP(!conp->icom_connected, -1, - dprintf("ibcom_put_scratch_pad,not connected\n")); - IBCOM_ERR_CHKANDJUMP(!sz, -1, dprintf("ibcom_put_scratch_pad,sz==0\n")); - - /* register memory area containing data */ - struct ibv_mr *mr_data = ibcom_reg_mr_fetch(laddr, sz); - IBCOM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("ibcom_put_scratch_pad,ibv_reg_mr_fetch failed\n")); - dprintf("ibcom_put_scratch_pad,"); - -#ifdef HAVE_LIBDCFA - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].mic_addr = (uint64_t) laddr; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = - mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); -#else - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = (uint64_t) laddr; -#endif - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].length = sz; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0].lkey = mr_data->lkey; - - /* num_sge is defined in ibcomOpen */ - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr_id = wr_id; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.atomic.remote_addr = - (uint64_t) conp->icom_rmem[IBCOM_SCRATCH_PAD_TO] + offset; - /* rkey is defined in ibcom_reg_mr_connect */ - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.atomic.compare_add = compare; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.atomic.swap = swap; - - dprintf("ibcom_put_scratch_pad,wr.rdma.remote_addr=%llx\n", - (unsigned long long) conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr); - -#ifdef HAVE_LIBDCFA - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR]); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno)); -#else - ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR], &bad_wr); - IBCOM_ERR_CHKANDJUMP(ib_errno, -1, - dprintf("ibcom_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", - ib_errno, bad_wr)); -#endif - - conp->ncom_scratch_pad += 1; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} -#endif - -/* poll completion queue */ -int ibcom_poll_cq(int which_cq, struct ibv_wc *wc, int *result) -{ - int ibcom_errno = 0; - - switch (which_cq) { - case IBCOM_RC_SHARED_RCQ: - *result = ibv_poll_cq(rc_shared_rcq, 1, wc); - break; - case IBCOM_RC_SHARED_SCQ: - *result = ibv_poll_cq(rc_shared_scq, 1, wc); - break; - case IBCOM_UD_SHARED_RCQ: - *result = ibv_poll_cq(ud_shared_rcq, 1, wc); - break; - case IBCOM_UD_SHARED_SCQ: - *result = ibv_poll_cq(ud_shared_scq, 1, wc); - break; - } - - if (*result < 0) { - dprintf("ibcom_poll_cq,status=%08x,vendor_err=%08x,len=%d,opcode=%08x,wr_id=%016lx\n", - wc->status, wc->vendor_err, wc->byte_len, wc->opcode, wc->wr_id); - ibcom_errno = *result; - goto fn_fail; - } - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_reg_mr_connect(int condesc, void *rmem, int rkey) -{ - int ibcom_errno = 0; - IbCom *conp; - int i; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - switch (conp->open_flag) { - case IBCOM_OPEN_RC: - case IBCOM_OPEN_RC_LMT_PUT: - conp->icom_rmem[IBCOM_RDMAWR_TO] = rmem; - conp->icom_rkey[IBCOM_RDMAWR_TO] = rkey; - conp->icom_sr[IBCOM_SMT_NOINLINE].wr.rdma.rkey = conp->icom_rkey[IBCOM_RDMAWR_TO]; - for (i = 0; i < IBCOM_SMT_INLINE_NCHAIN; i++) { - conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].wr.rdma.rkey = - conp->icom_rkey[IBCOM_RDMAWR_TO]; - } - break; - - case IBCOM_OPEN_SCRATCH_PAD: - conp->icom_rmem[IBCOM_SCRATCH_PAD_TO] = rmem; - conp->icom_rkey[IBCOM_SCRATCH_PAD_TO] = rkey; - conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].wr.rdma.rkey = - conp->icom_rkey[IBCOM_SCRATCH_PAD_TO]; - break; - - default: - dprintf("invalid open_flag=%d\n", conp->open_flag); - break; - } - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_get_info_conn(int condesc, int key, void *out, uint32_t out_len) -{ - int ibcom_errno = 0; - IbCom *conp; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - - switch (key) { - case IBCOM_INFOKEY_QP_QPN: - memcpy(out, &conp->icom_qp->qp_num, out_len); - break; - case IBCOM_INFOKEY_PORT_LID: -#ifdef HAVE_LIBDCFA - dprintf("ibcom_get_info_conn,lid=%04x\n", ib_ctx->lid); - memcpy(out, &ib_ctx->lid, out_len); -#else - dprintf("ibcom_get_info_conn,lid=%04x\n", conp->icom_pattr.lid); - memcpy(out, &conp->icom_pattr.lid, out_len); -#endif - break; - case IBCOM_INFOKEY_PORT_GID: - memcpy(out, &conp->icom_gid, out_len); - break; - case IBCOM_INFOKEY_PATTR_MAX_MSG_SZ:{ -#ifdef HAVE_LIBDCFA - uint32_t max_msg_sz = 1073741824; /* ConnectX-3 */ - memcpy(out, &max_msg_sz, out_len); -#else - memcpy(out, &conp->icom_pattr.max_msg_sz, out_len); -#endif - break; - } - default: - ibcom_errno = -1; - break; - } - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_get_info_mr(int condesc, int memid, int key, void *out, int out_len) -{ - int ibcom_errno = 0; - IbCom *conp; - struct ibv_mr *mr; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - IBCOM_ERR_CHKANDJUMP(memid >= conp->icom_mrlen, -1, - dprintf("ibcom_get_info_mr,wrong mem_id=%d\n", memid)); - mr = conp->icom_mrlist[memid]; - - switch (key) { - case IBCOM_INFOKEY_MR_ADDR: -#ifdef HAVE_LIBDCFA - /* host_addr is created by ibv_reg_mr in ibcomOpen, */ - /* dcfa_init read this host-addr, put it into KVS, the counter-party read it through KVS */ - memcpy(out, &mr->host_addr, out_len); -#else - memcpy(out, &mr->addr, out_len); -#endif - break; - case IBCOM_INFOKEY_MR_LENGTH:{ -#ifdef HAVE_LIBDCFA - assert(out_len == sizeof(size_t)); - size_t length = mr->size; /* type of mr->size is int */ - memcpy(out, &length, out_len); -#else - memcpy(out, &mr->length, out_len); -#endif - break; - } - case IBCOM_INFOKEY_MR_RKEY: - memcpy(out, &mr->rkey, out_len); - break; - default: - dprintf("ibcom_get_info_mr,unknown key=%d\n", key); - ibcom_errno = -1; - break; - } - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_mem_rdmawr_from(int condesc, void **out) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = - (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_FROM] + - IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_mem_rdmawr_to(int condesc, int seq_num, void **out) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZSEG * (seq_num % IBCOM_RDMABUF_NSEG); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_mem_udwr_from(int condesc, void **out) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = conp->icom_mem[IBCOM_UDWR_FROM]; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_mem_udwr_to(int condesc, void **out) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = conp->icom_mem[IBCOM_UDWR_TO]; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_sseq_num_get(int condesc, int *seq_num) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *seq_num = conp->sseq_num; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_lsr_seq_num_tail_get(int condesc, int **seq_num) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *seq_num = &(conp->lsr_seq_num_tail); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_rsr_seq_num_tail_get(int condesc, int **seq_num) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *seq_num = &(conp->rsr_seq_num_tail); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_rsr_seq_num_tail_last_sent_get(int condesc, int **seq_num) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *seq_num = &(conp->rsr_seq_num_tail_last_sent); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_rdmabuf_occupancy_notify_rate_get(int condesc, int *notify_rate) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - - switch (conp->rdmabuf_occupancy_notify_lstate) { - case IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW: - *notify_rate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW; - break; - case IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW: - *notify_rate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW; - break; - default: - ibcom_errno = -1; - goto fn_fail; - break; - } - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_rdmabuf_occupancy_notify_rstate_get(int condesc, int **rstate) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *rstate = &(conp->rdmabuf_occupancy_notify_rstate); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_rdmabuf_occupancy_notify_lstate_get(int condesc, int **lstate) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *lstate = &(conp->rdmabuf_occupancy_notify_lstate); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_obtain_pointer(int condesc, IbCom ** ibcom) -{ - IbCom *conp; - int ibcom_errno = 0; - - RANGE_CHECK_WITH_ERROR(condesc, conp); - *ibcom = conp; - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -static void ibcomShow(int condesc) -{ - IbCom *conp; - uint8_t *p; - int i; - - RANGE_CHECK(condesc, conp); - fprintf(stdout, "qp_num = %d\n", conp->icom_qp->qp_num); -#ifdef HAVE_LIBDCFA - fprintf(stdout, "lid = %d\n", ib_ctx->lid); -#else - fprintf(stdout, "lid = %d\n", conp->icom_pattr.lid); -#endif - p = (uint8_t *) & conp->icom_gid; - fprintf(stdout, "gid = %02x", p[0]); - for (i = 1; i < 16; i++) { - fprintf(stdout, ":%02x", p[i]); - } - fprintf(stdout, "\n"); -} - -static char *strerror_tbl[] = { - [0] = "zero", - [1] = "one", - [2] = "two", - [3] = "three", -}; - -char *ibcom_strerror(int errno) -{ - char *r; - if (-errno > 3) { - r = malloc(256); - sprintf(r, "%d", -errno); - goto fn_exit; - } - else { - r = strerror_tbl[-errno]; - } - fn_exit: - return r; - //fn_fail: - goto fn_exit; -} - -int ibcom_reg_mr(void *addr, int len, struct ibv_mr **mr) -{ - int ibcom_errno = 0; - dprintf("ibcom_reg_mr,addr=%p,len=%d,mr=%p\n", addr, len, mr); - - *mr = - ibv_reg_mr(ib_pd, addr, len, - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ); - - IBCOM_ERR_CHKANDJUMP(*mr == 0, -1, dprintf("ibcom_reg_mr,cannot register memory\n")); - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} - -int ibcom_dereg_mr(struct ibv_mr *mr) -{ - int i; - int ib_errno; - int ibcom_errno = 0; - - if (!mr) { - goto fn_exit; - } - - ib_errno = ibv_dereg_mr(mr); - if (ib_errno < 0) { - fprintf(stderr, "cannot deregister memory\n"); - goto fn_fail; - } -#ifdef HAVE_LIBDCFA - dprintf("ibcom_dereg_mr, addr=%p\n", mr->buf); -#else - dprintf("ibcom_dereg_mr, addr=%p\n", mr->addr); -#endif - - fn_exit: - return ibcom_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h deleted file mode 100644 index 0935205..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h +++ /dev/null @@ -1,423 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * (C) 2012 Oct 14 Yutaka Ishikawa, ishikawa(a)is.s.u-tokyo.ac.jp - * See COPYRIGHT in top-level directory. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <stdint.h> -#include <sys/types.h> -#include "mpid_nem_impl.h" - -//#define DEBUG_ON 1 -#ifdef DEBUG_ON -#define DEBUG if (dflag) -#else -#define DEBUG if (0) -#endif - -#ifdef HAVE_LIBDCFA -#include "dcfa.h" - -/* -*** diff -p verbs.h dcfa.h (structures) -same name, same fields - struct ibv_device { }; - struct ibv_context { }; - struct ibv_pd { }; - struct ibv_ah_attr { }; - -same name, different fields - struct ibv_qp_init_attr { -- void *qp_context; -- struct ibv_xrc_domain *xrc_domain; -}; - - struct ibv_mr { -- void *addr; -+ void *buf; -+ uint64_t host_addr; -- size_t length; -+ int size; -- uint32_t handle; -+ uint64_t handle; -+ int flag; 1: offload -- uint32_t lkey; -+ int lkey; -- uint32_t rkey; -+ int rkey; -}; - - struct ibv_qp { -+ struct mlx4_buf buf; -+ int max_inline_data; -+ int buf_size; - -+ uint32_t doorbell_qpn; -+ uint32_t sq_signal_bits; -+ int sq_spare_wqes; -+ struct mlx4_wq sq; - -+ uint32_t *db; // doorbell addr for post recv -+ struct mlx4_wq rq; -+ ibmic_qp_conn_info_t remote_qp_info; - -- uint32_t handle; -+ uint64_t handle; - -- struct ibv_context *context; -- void *qp_context; -- uint32_t events_completed; -- struct ibv_xrc_domain *xrc_domain; -- pthread_mutex_t mutex; -- pthread_cond_t cond; -}; - - struct ibv_cq { -- struct ibv_comp_channel *channel; -- void *cq_context; -- uint32_t handle; -- uint32_t comp_events_completed; -- uint32_t async_events_completed; - -- pthread_mutex_t mutex; -- pthread_cond_t cond; - -+ struct mlx4_buf buf; -+ uint32_t cons_index; -+ uint32_t wait_index; -+ uint32_t *set_ci_db; -+ uint32_t *arm_db; -+ int arm_sn; -+ int cqe_size; -+ uint64_t handle; -}; - - struct ibv_wc { -- uint32_t src_qp; -- uint16_t pkey_index; -- uint16_t slid; -- uint8_t sl; -- uint8_t dlid_path_bits; -}; - - struct ibv_send_wr { -- struct ibv_sge *sg_list; -+ struct ibv_sge sg_list[WR_SG_NUM]; -+ uint64_t addr; -+ uint32_t length; -+ uint32_t lkey; - }; - - struct ibv_recv_wr { -- struct ibv_sge *sg_list; -+ struct ibv_sge sg_list[WR_SG_NUM]; - }; - - struct ibv_sge { -+ uint64_t mic_addr; // buffer address on mic - }; - -non-existent -- struct ibv_port_attr { }; - - -*** diff -p verbs.h dcfa.h (functions) - -same name, same arguments - ibv_get_device_list - ibv_open_device - ibv_close_device - ibv_free_device_list - ibv_alloc_pd - ibv_dealloc_pd - ibv_create_qp - ibv_destroy_qp - ibv_reg_mr - ibv_dereg_mr - ibv_destroy_cq - ibv_poll_cq - ibv_modify_qp - -same name, different arguments -- int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr) -+ int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr); - -- int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) -+ int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr); - -- struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector); -+ struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe_max); - -non-existent -- ibv_get_device_name -- ibv_query_port -- ibv_query_gid -- ibv_create_ah -struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); -*/ - -#else -/* Original Infiniband */ -#include <infiniband/verbs.h> -#endif - -static inline unsigned long long getCPUCounter(void) -{ - unsigned int lo, hi; - __asm__ __volatile__(// serialize - "xorl %%eax,%%eax \n cpuid":::"%rax", "%rbx", "%rcx", "%rdx"); - __asm__ __volatile__("rdtsc":"=a"(lo), "=d"(hi)); - return (unsigned long long) hi << 32 | lo; -} - -extern struct ibv_cq *rc_shared_scq; -extern struct ibv_cq *rc_shared_scq_lmt_put; -extern struct ibv_cq *rc_shared_scq_scratch_pad; - -#define IBCOM_SIZE 2048 /* one process uses 2-4 fds */ -#define IBCOM_INLINE_DATA (512-64) /* experimented max is 884 */ /* this is lower bound and more than this value is set. the more this value is, the more the actual value set is. you need to check it */ - -#define IBCOM_MAX_SQ_CAPACITY (256/1) -#define IBCOM_MAX_RQ_CAPACITY ((IBCOM_MAX_SQ_CAPACITY)+16) /* We pre-post_recv IBCOM_MAX_SQ_CAPACITY of commands */ -#define IBCOM_MAX_SGE_CAPACITY (32/2) /* maximum for ConnectX-3 looks like 32 */ -#define IBCOM_MAX_CQ_CAPACITY IBCOM_MAX_RQ_CAPACITY -#define IBCOM_MAX_CQ_HEIGHT_DRAIN (((IBCOM_MAX_CQ_CAPACITY)>>2)+((IBCOM_MAX_CQ_CAPACITY)>>1)) /* drain when reaching this amount */ -#define IBCOM_MAX_SQ_HEIGHT_DRAIN (((IBCOM_MAX_SQ_CAPACITY)>>2)+((IBCOM_MAX_SQ_CAPACITY)>>1)) /* drain when reaching this amount */ -#define IBCOM_AMT_CQ_DRAIN ((IBCOM_MAX_CQ_CAPACITY)>>2) /* drain this amount */ -#define IBCOM_MAX_RD_ATOMIC 4 - -#define IBCOM_MAX_TRIES 1 -#define IBCOM_SCQ_FLG 1 -#define IBCOM_RCQ_FLG 2 - -#define IBCOM_INFOKEY_PATTR_MAX_MSG_SZ 100 -#define IBCOM_INFOKEY_MR_ADDR 200 -#define IBCOM_INFOKEY_MR_LENGTH 201 -#define IBCOM_INFOKEY_MR_RKEY 202 -#define IBCOM_INFOKEY_QP_QPN 300 -#define IBCOM_INFOKEY_PORT_LID 400 -#define IBCOM_INFOKEY_PORT_GID 401 - - -/* buffers */ -#define IBCOM_NBUF_RDMA 2 /* number of <addr, sz, lkey, rkey> */ -#define IBCOM_RDMAWR_FROM 0 /* index to RDMA-write-from buffer */ -#define IBCOM_RDMAWR_TO 1 /* index to RDMA-write-to buffer */ -/* assuming that the unit (32768) is equals to eager-RDMA-write threashold - assuming that the multiplier (256) is - equals to max number of outstanding eager-RDMA-write transactions */ -#define IBCOM_RDMABUF_SZSEG (16384/4) //(16384+8+40+1) /* this size minus magics and headers must be 2^n because data might grow to the next 2^m boundary, see dcfa_impl.h, dcfa_ibcom.c, src/mpid/ch3/src/mpid_isend.c */ -#define IBCOM_RDMABUF_SZ ((IBCOM_RDMABUF_SZSEG) * 16) /* (32768 * 256) */ -#define IBCOM_RDMABUF_NSEG ((IBCOM_RDMABUF_SZ) / (IBCOM_RDMABUF_SZSEG)) -#define IBCOM_SMT_INLINE_NCHAIN 8 /* maximum number of chained inline-send commands */ -#define IBCOM_RDMABUF_HIGH_WATER_MARK (((IBCOM_RDMABUF_NSEG)>>1)+((IBCOM_RDMABUF_NSEG)>>2)) -#define IBCOM_RDMABUF_LOW_WATER_MARK (((IBCOM_RDMABUF_NSEG)>>2)) -#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW 1 -#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW 2 -#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW /*1*/(((IBCOM_RDMABUF_NSEG)>>4) == 0 ? 1 : ((IBCOM_RDMABUF_NSEG)>>4)) -#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((IBCOM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */ -#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */ - -#define IBCOM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */ -#define IBCOM_UDWR_FROM 0 /* index to UD-write-from buffer */ -#define IBCOM_UDWR_TO 1 /* index to UD-write-to buffer */ -#define IBCOM_UDBUF_SZ (128 * 8192) /* supporting 100K ranks with 10 rounds */ -#define IBCOM_UDBUF_SZSEG (128) -#define IBCOM_UDBUF_NSEG (IBCOM_UDBUF_SZ / IBCOM_UDBUF_SZSEG) - -#define IBCOM_NBUF_SCRATCH_PAD 1 /* number of <addr, sz, lkey, rkey> */ -#define IBCOM_SCRATCH_PAD_TO 0 /* index to RDMA-write-to buffer */ - -/* send command templates */ -#define IBCOM_RC_SR_NTEMPLATE (8+1+2) /* number of request templates, 8 for inline-chained-smt, 1 for smt, 1 for lmt */ -#define IBCOM_SMT_INLINE_CHAINED0 0 /* index to it */ -#define IBCOM_SMT_INLINE_CHAINED7 7 -#define IBCOM_SMT_NOINLINE 8 -#define IBCOM_LMT_INITIATOR 9 /* FIXME: bad naming */ - -#define IBCOM_RC_SR_LMT_PUT_NTEMPLATE IBCOM_RC_SR_NTEMPLATE /* FIXME: TEMPLATE named IBCOM_RC_SR shares IBCOM_LMT_PUT */ -#define IBCOM_LMT_PUT 10 - -/* recv command templates */ -#define IBCOM_RC_RR_NTEMPLATE 1 /* 1 for smt, */ -#define IBCOM_RDMAWR_RESPONDER 0 /* index to recv request template */ - -/* sge template */ -#define IBCOM_SMT_INLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ -#define IBCOM_SMT_NOINLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ -#define IBCOM_LMT_INITIATOR_NSGE 1 /* data x1 */ -#define IBCOM_LMT_PUT_NSGE 1 /* data x1 */ -#define IBCOM_SCRATCH_PAD_INITIATOR_NSGE 1 /* QP state */ - -#define IBCOM_UD_SR_NTEMPLATE 1 -#define IBCOM_UD_RR_NTEMPLATE 1 -#define IBCOM_UD_INITIATOR 0 /* index to send request template */ -#define IBCOM_UD_RESPONDER 0 /* index to recv request template */ - -#define IBCOM_SCRATCH_PAD_SR_NTEMPLATE 2 -#define IBCOM_SCRATCH_PAD_RR_NTEMPLATE 1 -#define IBCOM_SCRATCH_PAD_INITIATOR 0 /* index to send request template */ -#define IBCOM_SCRATCH_PAD_CAS 1 -#define IBCOM_SCRATCH_PAD_RESPONDER 0 /* index to recv request template */ - - -typedef struct IbCom { - short icom_used; - short icom_connected; - int icom_port; -#ifdef HAVE_LIBDCFA -#else - struct ibv_port_attr icom_pattr; /* IB port attributes */ -#endif - struct ibv_qp *icom_qp; - struct ibv_cq *icom_scq; - struct ibv_cq *icom_rcq; - struct ibv_mr **icom_mrlist; - int icom_mrlen; - union ibv_gid icom_gid; - void **icom_mem; /* 0: send 1: recv 2..: rdma */ - int *icom_msize; /* 0: send 1: recv 2..: rdma */ - struct ibv_send_wr *icom_sr; - struct ibv_ah_attr *icom_ah_attr; - struct ibv_recv_wr *icom_rr; - void **icom_rmem; - int *icom_rkey; - size_t *icom_rsize; - int sseq_num; - int rsr_seq_num_poll; - int rsr_seq_num_tail; /* occupation status of remote Send Request (SR) queue (it covers occupation status of local RDMA-wr-to buffer) */ - int rsr_seq_num_tail_last_sent; /* latest one sent to remote rank */ - int lsr_seq_num_tail; /* occupation status of local Send Request (SR) queue */ - int lsr_seq_num_tail_last_requested; /* value when lmt_start_send issued req_seq_num */ - int rdmabuf_occupancy_notify_rstate, rdmabuf_occupancy_notify_lstate; - int ncom, ncom_lmt_put, ncom_scratch_pad; /* number of entries in the command queue */ - - uint32_t max_inline_data; /* actual value obtained after ibv_create_qp */ - uint32_t max_send_wr; - uint32_t max_recv_wr; - - uint32_t open_flag; /* IBCOM_OPEN_UD, ... */ - uint16_t remote_lid; /* for debug */ - - /* other commands can be executed before RDMA-rd command */ - /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ - uint16_t after_rdma_rd; - - uint64_t rsr_seq_num_released[(IBCOM_RDMABUF_NSEG + 63) / 64]; - -} IbCom; - -extern int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc); -extern int ibcom_alloc(int condesc, int sz); -extern int ibcom_close(int); -extern int ibcom_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid); - -extern int ibcom_reg_mr_connect(int condesc, void *rmem, int rkey); -extern int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, - int sz_hdr, void *data, int sz_data, int *copied); -extern int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, - int sz_data); -extern int ibcom_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr); -//extern int ibcom_isend(int condesc, uint64_t wr_id, void* hdr, int sz_hdr, void* data, int sz_data); -extern int ibcom_irecv(int condesc, uint64_t wr_id); -extern int ibcom_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, - uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id); -extern int ibcom_udrecv(int condesc); -extern int ibcom_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, - void *laddr); -extern int ibcom_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, - void *laddr); -extern int ibcom_poll_cq(int which_cq, struct ibv_wc *wc, int *result); - -extern int ibcom_obtain_pointer(int condesc, IbCom ** ibcom); - -/* for dcfa_reg_mr.c */ -extern int ibcom_reg_mr(void *addr, int len, struct ibv_mr **mr); -extern int ibcom_dereg_mr(struct ibv_mr *mr); - -extern int ibcom_get_info_conn(int condesc, int key, void *out, uint32_t out_len); -extern int ibcom_get_info_mr(int condesc, int memid, int key, void *out, int out_len); - -extern int ibcom_sseq_num_get(int condesc, int *seq_num); -extern int ibcom_lsr_seq_num_tail_get(int condesc, int **seq_num); -extern int ibcom_rsr_seq_num_tail_get(int condesc, int **seq_num); -extern int ibcom_rsr_seq_num_tail_last_sent_get(int condesc, int **seq_num); -extern int ibcom_rdmabuf_occupancy_notify_rate_get(int condesc, int *notify_rate); -extern int ibcom_rdmabuf_occupancy_notify_rstate_get(int condesc, int **rstate); -extern int ibcom_rdmabuf_occupancy_notify_lstate_get(int condesc, int **lstate); - -extern int ibcomMemInfo(int, int, void **, size_t *, int *); -extern char *ibcom_strerror(int errno); -extern int dflag; - -extern int ibcom_mem_rdmawr_from(int condesc, void **out); -extern int ibcom_mem_rdmawr_to(int condesc, int seq_num, void **out); -extern int ibcom_mem_udwr_from(int condesc, void **out); -extern int ibcom_mem_udwr_to(int condesc, void **out); - -/* dcfa_reg_mr.c */ -extern void ibcom_RegisterCacheInit(void); -extern void ibcom_RegisterCacheDestroy(void); -extern struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len); - -/* dcfa_ctlmsg.c */ -extern int ibcom_udbuf_init(void *q); - -#define IBCOM_RC_SHARED_RCQ 0 -#define IBCOM_RC_SHARED_SCQ 1 -#define IBCOM_UD_SHARED_RCQ 2 -#define IBCOM_UD_SHARED_SCQ 3 -#define IBCOM_RC_SHARED_SCQ_LMT_PUT 4 - -/* flag for open */ -#define IBCOM_OPEN_RC 0x01 -/* for MPI control message, eager send, rendezvous protocol, - so via RC-send/recv or RDMA-write/RDMA-read */ - -#define IBCOM_OPEN_UD 0x02 -/* obsolete, to wait for you-to-me QP to become RTR state - so via UD-send/recv */ - -#define IBCOM_OPEN_RC_LMT_PUT 0x03 -/* obsolete, tried to use different CQ for LMT-PUT protocol for speed */ - -#define IBCOM_OPEN_SCRATCH_PAD 0x04 -/* obsolete, to wait for you-to-me QP to become RTR state - so via RDMA-write */ - -#define IBCOM_ERR_SETANDJUMP(errno, stmt) { stmt; ibcom_errno = errno; goto fn_fail; } -#define IBCOM_ERR_CHKANDJUMP(cond, errno, stmt) if (cond) { stmt; ibcom_errno = errno; goto fn_fail; } - -#define IBCOM_QKEY 0x1234 -#define IBCOM_MAGIC 0x55 - -typedef struct sz_hdrmagic_t { - uint32_t sz; - uint32_t magic; -} sz_hdrmagic_t; - - -typedef struct tailmagic_t { - uint8_t magic; - //uint32_t traits; /* for debug */ -} tailmagic_t; - -#define DCFA_NEM_SZ_DATA_POW2(sz) \ - for(sz_data_pow2 = 15; sz_data_pow2 < (sz); sz_data_pow2 = ((((sz_data_pow2 + 1) << 1) - 1) > IBCOM_RDMABUF_SZSEG - sizeof(tailmagic_t)) ? IBCOM_RDMABUF_SZSEG - sizeof(tailmagic_t) : (((sz_data_pow2 + 1) << 1) - 1)) { } \ - if (sz_data_pow2 > IBCOM_RDMABUF_SZSEG - sizeof(tailmagic_t)) { printf("assertion failed\n"); }; \ - -#define DCFA_NEM_MAX_DATA_POW2 (IBCOM_RDMABUF_SZSEG - sizeof(tailmagic_t)) - -typedef struct ibcom_qp_state_t { - uint32_t state; -} ibcom_qp_state_t; - -#define IBCOM_QP_STATE_RTR 0x12345678 -#define IBCOM_SZ_MPI_HEADER 48 -#define IBCOM_AMT_SLACK (IBCOM_RDMABUF_NSEG > 128 ? 1 : 1) diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h deleted file mode 100644 index 9922d0c..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h +++ /dev/null @@ -1,645 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * See COPYRIGHT in top-level directory. - */ - -#ifndef DCFA_IMPL_H -//#define DCFA_IMPL_H - -#include "mpid_nem_impl.h" -#include "dcfa_ibcom.h" -#include <sys/types.h> -#include <errno.h> -#include <linux/mman.h> /* make it define MAP_ANONYMOUS */ -#include <sys/mman.h> - -#define LMT_GET_CQE /* detect RDMA completion by CQE */ -#define DISABLE_VAR_OCC_NOTIFY_RATE -/* lmt-put: - (1) receiver sends cts to sender (2) sender RDMA-write to receiver - (3) sender fetch CQE (4) receiver polls on end-flag -*/ -//#define DCFA_ONDEMAND - -typedef struct { - union ibv_gid gid; - uint16_t lid; - uint32_t qpn; -} dcfa_conn_ud_t; - -typedef struct { - int fd, fd_lmt_put; - MPIDI_VC_t *vc; -} dcfaconn_t; - -/* see src/mpid/ch3/channels/nemesis/include/mpid_nem_generic_queue.h */ -typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_dcfa_sendq_t; - -/* The vc provides a generic buffer in which network modules can store - private fields This removes all dependencies from the VC struction - on the network module, facilitating dynamic module loading. */ -typedef struct { - dcfaconn_t *sc; - int pending_sends; /* number of send in flight */ - IbCom *ibcom, *ibcom_lmt_put; - MPID_nem_dcfa_sendq_t sendq; /* overflow queue for IB commands */ - MPID_nem_dcfa_sendq_t sendq_lmt_put; - int is_connected; /* dynamic connection, checked in iSendContig, protocol processed there and in progress engine */ -} MPID_nem_dcfa_vc_area; - -/* macro for secret area in vc */ -#define VC_CH(vc) ((MPIDI_CH3I_VC *)&(vc)->ch) -#define VC_DCFA(vc) ((MPID_nem_dcfa_vc_area *)VC_CH((vc))->netmod_area.padding) -#define VC_FIELD(vcp, field) (((MPID_nem_dcfa_vc_area *)VC_CH(((vcp)))->netmod_area.padding)->field) - -/* The req provides a generic buffer in which network modules can store - private fields This removes all dependencies from the req structure - on the network module, facilitating dynamic module loading. */ -typedef struct { - int seq_num; /* NOT USED, DELETE IT: sequence number of SR which RDMA-RD for lmt releases in dcfa_poll */ - struct MPID_Request *lmt_next; /* for lmtq */ - struct MPID_Request *sendq_next; /* for sendq */ - void *lmt_raddr; /* remember this for sendq, it might be better to use sreq->dev.iov[0].MPID_IOV_BUF instead */ - uint32_t lmt_rkey; /* remember this for sendq, survive over lrecv and referenced when dequeueing from sendq */ - uint32_t lmt_szsend; /* remember this for sendq */ - uint8_t lmt_tail, lmt_sender_tail, lmt_receiver_tail; /* survive over lrecv and referenced when polling */ - MPI_Aint lmt_dt_true_lb; /* to locate the last byte of receive buffer */ - void *lmt_write_to_buf; /* user buffer or temporary buffer for pack and remember it for lmt_orderq */ - void *lmt_pack_buf; /* to pack non-contiguous data */ -} MPID_nem_dcfa_req_area; - -/* macro for secret area in req */ -#define REQ_DCFA(req) ((MPID_nem_dcfa_req_area *)(&(req)->ch.netmod_area.padding)) -#define REQ_FIELD(reqp, field) (((MPID_nem_dcfa_req_area *)((reqp)->ch.netmod_area.padding))->field) - -/* see src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h */ -/* sreq is never enqueued into posted-queue nor unexpected-queue, so we can reuse sreq->dev.next */ -#define MPID_nem_dcfa_sendq_empty(q) GENERICM_Q_EMPTY (q) -#define MPID_nem_dcfa_sendq_head(q) GENERICM_Q_HEAD (q) -#define MPID_nem_dcfa_sendq_next_field(ep, next_field) REQ_FIELD(ep, next_field) -#define MPID_nem_dcfa_sendq_next(ep) REQ_FIELD(ep, sendq_next) -//#define MPID_nem_dcfa_sendq_next(ep) (ep->dev.next) /*takagi*/ -#define MPID_nem_dcfa_sendq_enqueue(qp, ep) GENERICM_Q_ENQUEUE (qp, ep, MPID_nem_dcfa_sendq_next_field, sendq_next); -#define MPID_nem_dcfa_sendq_enqueue_at_head(qp, ep) GENERICM_Q_ENQUEUE_AT_HEAD(qp, ep, MPID_nem_dcfa_sendq_next_field, sendq_next); -#define MPID_nem_dcfa_sendq_dequeue(qp, ep) GENERICM_Q_DEQUEUE (qp, ep, MPID_nem_dcfa_sendq_next_field, sendq_next); - -/* see src/mpid/ch3/channels/nemesis/include/mpid_nem_generic_queue.h */ -typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_dcfa_lmtq_t; - -/* connection manager */ -typedef struct { - int remote_rank; - uint32_t type; /* SYN */ - uint32_t qpn; /* QPN for eager-send channel */ - uint32_t rkey; /* key for RDMA-write-to buffer of eager-send channel */ - void *rmem; /* address of RDMA-write-to buffer of eager-send channel */ -} MPID_nem_dcfa_cm_pkt_syn_t; - -typedef struct { - int remote_rank; - uint32_t type; /* SYNACK */ - uint32_t qpn; /* QPN for eager-send channel */ - uint32_t rkey; /* key for RDMA-write-to buffer of eager-send channel */ - void *rmem; /* address of RDMA-write-to buffer of eager-send channel */ -} MPID_nem_dcfa_cm_pkt_synack_t; - -typedef union { - MPID_nem_dcfa_cm_pkt_syn_t syn; - MPID_nem_dcfa_cm_pkt_synack_t synack; -} MPID_nem_dcfa_cm_pkt_t; - -typedef struct MPID_nem_dcfa_cm_sendq_entry { - MPID_nem_dcfa_cm_pkt_t pending_pkt; - struct MPID_nem_dcfa_cm_sendq_entry *sendq_next; /* for software command queue */ -} MPID_nem_dcfa_cm_sendq_entry_t; - -#ifdef DCFA_ONDEMAND -typedef struct { - char *data; - int length; - int max_length; -} MPID_nem_dcfa_cm_map_t; - -typedef struct { - uint32_t type; - uint32_t qpnum; - uint16_t lid; - union ibv_gid gid; - void *rmem; - uint32_t rkey; -} MPID_nem_dcfa_cm_cmd_t; -#endif - -typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_dcfa_cm_sendq_t; - -#define MPID_nem_dcfa_cm_sendq_empty(q) GENERICM_Q_EMPTY (q) -#define MPID_nem_dcfa_cm_sendq_head(q) GENERICM_Q_HEAD (q) -#define MPID_nem_dcfa_cm_sendq_next_field(ep, next_field) ((ep)->next_field) -#define MPID_nem_dcfa_cm_sendq_next(ep) ((ep)->sendq_next) -#define MPID_nem_dcfa_cm_sendq_enqueue(qp, ep) GENERICM_Q_ENQUEUE (qp, ep, MPID_nem_dcfa_cm_sendq_next_field, sendq_next); -#define MPID_nem_dcfa_cm_sendq_enqueue_at_head(qp, ep) GENERICM_Q_ENQUEUE_AT_HEAD(qp, ep, MPID_nem_dcfa_cm_sendq_next_field, sendq_next); -#define MPID_nem_dcfa_cm_sendq_dequeue(qp, ep) GENERICM_Q_DEQUEUE (qp, ep, MPID_nem_dcfa_cm_sendq_next_field, sendq_next); - -/* see src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h */ -/* TODO: rreq for rendezvous is dequeued from posted-queue nor unexpected-queue when do_cts is called, - so we can reuse rreq->dev.next */ -#define MPID_nem_dcfa_lmtq_empty(q) GENERICM_Q_EMPTY(q) -#define MPID_nem_dcfa_lmtq_head(q) GENERICM_Q_HEAD(q) -#define MPID_nem_dcfa_lmtq_next_field(ep, next_field) REQ_FIELD(ep, next_field) -#define MPID_nem_dcfa_lmtq_next(ep) REQ_FIELD(ep, lmt_next) -#define MPID_nem_dcfa_lmtq_enqueue(qp, ep) GENERICM_Q_ENQUEUE(qp, ep, MPID_nem_dcfa_lmtq_next_field, lmt_next); - -#define MPID_nem_dcfa_diff32(a, b) ((uint32_t)((a + (1ULL<<32) - b) & ((1ULL<<32)-1))) -#define MPID_nem_dcfa_sendq_ready_to_send_head(vc_dcfa) (vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY && MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY && MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG) -#define MPID_nem_dcfa_sendq_ready_to_send_head_lmt_put(vc_dcfa) (vc_dcfa->ibcom->ncom_lmt_put < IBCOM_MAX_SQ_CAPACITY && MPID_nem_dcfa_ncqe_lmt_put < IBCOM_MAX_CQ_CAPACITY) - -/* counting bloom filter to detect multiple lmt-sends in one send-wait period to - avoid overwriting the last byte in the receive buffer */ -#define MPID_nem_dcfa_cbf_nslot 16 /* slots */ -#define MPID_nem_dcfa_cbf_bitsperslot 4 /* one slot can accomodate multiple bits */ -#define MPID_nem_dcfa_cbf_lognslot 4 -#define MPID_nem_dcfa_cbf_nhash 3 /* number of hash functions */ -#define MPID_nem_dcfa_getpos \ - int pos_8b = pos / (8 / MPID_nem_dcfa_cbf_bitsperslot);\ - assert(0 <= pos_8b && pos_8b < MPID_nem_dcfa_cbf_nslot * MPID_nem_dcfa_cbf_bitsperslot / 8);\ - int pos_bps = pos & (8 / MPID_nem_dcfa_cbf_bitsperslot - 1); -#define MPID_nem_dcfa_shift \ - ((array[pos_8b] >> (pos_bps * MPID_nem_dcfa_cbf_bitsperslot)) & ((1ULL<<MPID_nem_dcfa_cbf_bitsperslot) - 1)) -#define MPID_nem_dcfa_maskset \ - array[pos_8b] &= ~(((1ULL<<MPID_nem_dcfa_cbf_bitsperslot) - 1) << (pos_bps * MPID_nem_dcfa_cbf_bitsperslot)); \ - array[pos_8b] |= (bits & ((1ULL<<MPID_nem_dcfa_cbf_bitsperslot)-1)) << (pos_bps * MPID_nem_dcfa_cbf_bitsperslot) -static inline int MPID_nem_dcfa_cbf_get(uint8_t * array, int pos) -{ - MPID_nem_dcfa_getpos; - return MPID_nem_dcfa_shift; -} - -static inline void MPID_nem_dcfa_cbf_set(uint8_t * array, int pos, uint16_t bits) -{ - MPID_nem_dcfa_getpos; - MPID_nem_dcfa_maskset; -} - -static inline void MPID_nem_dcfa_cbf_inc(uint8_t * array, int pos) -{ - MPID_nem_dcfa_getpos; - int16_t bits = MPID_nem_dcfa_shift; - assert(bits != (1ULL << MPID_nem_dcfa_cbf_bitsperslot) - 1); - bits++; - MPID_nem_dcfa_maskset; -} - -static inline void MPID_nem_dcfa_cbf_dec(uint8_t * array, int pos) -{ - MPID_nem_dcfa_getpos; - int16_t bits = MPID_nem_dcfa_shift; - assert(bits != 0); - bits--; - MPID_nem_dcfa_maskset; -} - -static inline int MPID_nem_dcfa_cbf_hash1(uint64_t addr) -{ - return - (((addr >> (MPID_nem_dcfa_cbf_lognslot * 0)) & (MPID_nem_dcfa_cbf_nslot - 1)) ^ - ((addr >> (MPID_nem_dcfa_cbf_lognslot * 3)) & (MPID_nem_dcfa_cbf_nslot - 1)) ^ - ((addr >> (MPID_nem_dcfa_cbf_lognslot * 6)) & (MPID_nem_dcfa_cbf_nslot - 1)) - + 1) & (MPID_nem_dcfa_cbf_nslot - 1); -} - -static inline int MPID_nem_dcfa_cbf_hash2(uint64_t addr) -{ - /* adding one because addr tends to have a postfix of "fff" */ - return - (((addr >> (MPID_nem_dcfa_cbf_lognslot * 1)) & (MPID_nem_dcfa_cbf_nslot - 1)) ^ - ((addr >> (MPID_nem_dcfa_cbf_lognslot * 4)) & (MPID_nem_dcfa_cbf_nslot - 1)) ^ - ((addr >> (MPID_nem_dcfa_cbf_lognslot * 7)) & (MPID_nem_dcfa_cbf_nslot - 1)) - + 1) & (MPID_nem_dcfa_cbf_nslot - 1); -} - -static inline int MPID_nem_dcfa_cbf_hash3(uint64_t addr) -{ - /* adding two because addr tends to have a postfix of "fff" */ - return - (((addr >> (MPID_nem_dcfa_cbf_lognslot * 2)) & (MPID_nem_dcfa_cbf_nslot - 1)) ^ - ((addr >> (MPID_nem_dcfa_cbf_lognslot * 5)) & (MPID_nem_dcfa_cbf_nslot - 1)) ^ - ((addr >> (MPID_nem_dcfa_cbf_lognslot * 8)) & (MPID_nem_dcfa_cbf_nslot - 1)) - + 2) & (MPID_nem_dcfa_cbf_nslot - 1); - -} - -static inline void MPID_nem_dcfa_cbf_add(uint64_t addr, uint8_t * array) -{ - //dprintf("cbf_add,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_dcfa_cbf_hash1(addr), MPID_nem_dcfa_cbf_hash2(addr), MPID_nem_dcfa_cbf_hash3(addr)); - //dprintf("cbf_add,%d,%d,%d\n", MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr))); - MPID_nem_dcfa_cbf_inc(array, MPID_nem_dcfa_cbf_hash1(addr)); - MPID_nem_dcfa_cbf_inc(array, MPID_nem_dcfa_cbf_hash2(addr)); - MPID_nem_dcfa_cbf_inc(array, MPID_nem_dcfa_cbf_hash3(addr)); - //dprintf("cbf_add,%d,%d,%d\n", MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr))); -} - -static inline void MPID_nem_dcfa_cbf_delete(uint64_t addr, uint8_t * array) -{ - //dprintf("cbf_delete,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_dcfa_cbf_hash1(addr), MPID_nem_dcfa_cbf_hash2(addr), MPID_nem_dcfa_cbf_hash3(addr)); - //dprintf("cbf_delete,%d,%d,%d\n", MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr))); - MPID_nem_dcfa_cbf_dec(array, MPID_nem_dcfa_cbf_hash1(addr)); - MPID_nem_dcfa_cbf_dec(array, MPID_nem_dcfa_cbf_hash2(addr)); - MPID_nem_dcfa_cbf_dec(array, MPID_nem_dcfa_cbf_hash3(addr)); - //dprintf("cbf_delete,%d,%d,%d\n", MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr))); -} - -static inline int MPID_nem_dcfa_cbf_query(uint64_t addr, uint8_t * array) -{ - //dprintf("cbf_query,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_dcfa_cbf_hash1(addr), MPID_nem_dcfa_cbf_hash2(addr), MPID_nem_dcfa_cbf_hash3(addr)); - //dprintf("cbf_query,%d,%d,%d\n", MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr))); - return - MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)) > 0 && - MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)) > 0 && - MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr)) > 0; -} - -static inline int MPID_nem_dcfa_cbf_would_overflow(uint64_t addr, uint8_t * array) -{ - //dprintf("cbf_would_overflow,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_dcfa_cbf_hash1(addr), MPID_nem_dcfa_cbf_hash2(addr), MPID_nem_dcfa_cbf_hash3(addr)); - //dprintf("cbf_would_overflow,%d,%d,%d\n", MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash1(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash2(addr)), MPID_nem_dcfa_cbf_get(array, MPID_nem_dcfa_cbf_hash3(addr))); - return - MPID_nem_dcfa_cbf_get(array, - MPID_nem_dcfa_cbf_hash1(addr)) == - (1ULL << MPID_nem_dcfa_cbf_bitsperslot) - 1 || - MPID_nem_dcfa_cbf_get(array, - MPID_nem_dcfa_cbf_hash2(addr)) == - (1ULL << MPID_nem_dcfa_cbf_bitsperslot) - 1 || - MPID_nem_dcfa_cbf_get(array, - MPID_nem_dcfa_cbf_hash3(addr)) == - (1ULL << MPID_nem_dcfa_cbf_bitsperslot) - 1; -} - -/* functions */ -uint8_t MPID_nem_dcfa_rand(void); -uint64_t MPID_nem_dcfa_rdtsc(void); -int MPID_nem_dcfa_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p); -int MPID_nem_dcfa_finalize(void); -int MPID_nem_dcfa_drain_scq(int dont_call_progress); -int MPID_nem_dcfa_drain_scq_lmt_put(void); -int MPID_nem_dcfa_drain_scq_scratch_pad(void); -int MPID_nem_dcfa_poll(int in_blocking_poll); -int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc); - -int MPID_nem_dcfa_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p); -int MPID_nem_dcfa_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc); -int MPID_nem_dcfa_vc_init(MPIDI_VC_t * vc); -int MPID_nem_dcfa_vc_destroy(MPIDI_VC_t * vc); -int MPID_nem_dcfa_vc_terminate(MPIDI_VC_t * vc); -int MPID_nem_dcfa_pkthandler_init(MPIDI_CH3_PktHandler_Fcn * pktArray[], int arraySize); - -int MPID_nem_dcfa_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header, - MPIDI_msg_sz_t hdr_sz); - -/* CH3 send/recv functions */ -int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, - MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz); -int MPID_nem_dcfa_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, - MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr); - -/* used by dcfa_poll.c */ -int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa); - -/* CH3--lmt send/recv functions */ -int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, - struct MPID_Request *req); -int MPID_nem_dcfa_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, - void *write_to_buf); -int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie); -int MPID_nem_dcfa_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie); -int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req); -int MPID_nem_dcfa_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req); -int MPID_nem_dcfa_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *req); -int MPID_nem_dcfa_lmt_vc_terminated(struct MPIDI_VC *vc); -/* overriding functions - initialize the value of a member named "recv_posted" - in BSS-variable named "comm_ops" with type of MPIDI_Comm_ops_t - to "MPID_nem_dcfa_recv_posted" in dcfa_init.c - MPIDI_Comm_ops_t is defined in src/mpid/ch3/include/mpidimpl.h */ -int MPID_nem_dcfa_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req); -int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data); - -void pht_update(uint64_t vpc, uint32_t hist, int32_t taken); -int pht_pred(uint64_t vpc, uint32_t hist); - -/* Keys for business cards */ -#define MPIDI_CH3I_GID_KEY "gid" -#define MPIDI_CH3I_LID_KEY "lid" -#define MPIDI_CH3I_QPN_KEY "qpn" -#define MPIDI_CH3I_RKEY_KEY "rkey" -#define MPIDI_CH3I_RMEM_KEY "rmem" - -#define MPID_NEM_DCFA_RECV_MAX_PKT_LEN 1024 - -extern int MPID_nem_dcfa_conn_ud_fd; -extern IbCom *MPID_nem_dcfa_conn_ud_ibcom; -extern dcfa_conn_ud_t *MPID_nem_dcfa_conn_ud; -extern dcfaconn_t *MPID_nem_dcfa_conns; -extern MPIDI_VC_t **MPID_nem_dcfa_pollingset; -extern int *MPID_nem_dcfa_scratch_pad_fds; -extern int MPID_nem_dcfa_npollingset; -extern void *MPID_nem_dcfa_fl[18]; -extern int MPID_nem_dcfa_nranks; -//extern char *MPID_nem_dcfa_recv_buf; -extern int MPID_nem_dcfa_myrank; -extern uint64_t MPID_nem_dcfa_tsc_poll; /* to throttle dcfa_poll in recv_posted (in dcfa_poll.c) */ -extern int MPID_nem_dcfa_ncqe; /* for lazy poll scq */ -extern int MPID_nem_dcfa_ncqe_lmt_put; /* lmt-put uses another QP, SQ, CQ to speed-up fetching CQE */ -#ifdef DCFA_ONDEMAND -extern MPID_nem_dcfa_cm_map_t MPID_nem_dcfa_cm_state; -extern int MPID_nem_dcfa_ncqe_connect; /* couting outstanding connection requests */ -#endif -extern int MPID_nem_dcfa_ncqe_scratch_pad; -extern int MPID_nem_dcfa_ncqe_to_drain; /* count put in lmt-put-done protocol */ -extern int MPID_nem_dcfa_ncqe_nces; /* counting non-copied eager-send */ -extern MPID_nem_dcfa_lmtq_t MPID_nem_dcfa_lmtq; /* poll queue for lmt */ -extern MPID_nem_dcfa_lmtq_t MPID_nem_dcfa_lmt_orderq; /* force order when two or more rts_to_sender randomizes the last byte of receive buffer */ -extern MPID_nem_dcfa_vc_area *MPID_nem_dcfa_debug_current_vc_dcfa; - -/* to detect multiple lmt-sends in one send-wait period to - avoid overwriting the last byte in the receive buffer */ -extern uint8_t MPID_nem_dcfa_lmt_tail_addr_cbf[MPID_nem_dcfa_cbf_nslot * - MPID_nem_dcfa_cbf_bitsperslot / 8]; - -#define MPID_NEM_DCFA_MAX_POLLINGSET 65536 - -/* xfer.c manages memory region using memid */ -#define MPID_NEM_DCFA_MEMID_RDMA 0 - -/* command using IB UD */ -#define MPID_NEM_DCFA_SYNC_SYN 0 -#define MPID_NEM_DCFA_SYNC_SYNACK 1 -#define MPID_NEM_DCFA_SYNC_NACK 2 - -#define MPID_NEM_DCFA_EAGER_MAX_MSG_SZ (IBCOM_RDMABUF_SZSEG/*1024*/-sizeof(MPIDI_CH3_Pkt_t)+sizeof(MPIDI_CH3_Pkt_eager_send_t)-sizeof(sz_hdrmagic_t)-sizeof(MPID_nem_dcfa_pkt_prefix_t)-sizeof(tailmagic_t)) /* when > this size, lmt is used. see src/mpid/ch3/src/mpid_isend.c */ -#define MPID_NEM_DCFA_POLL_PERIOD_RECV_POSTED 2000 /* minimum period from previous dcfa_poll to dcfa_poll in recv_posted */ -#define MPID_NEM_DCFA_POLL_PERIOD_SEND_POSTED 2000 - -typedef struct { - void *addr; - uint32_t rkey; - int seq_num_tail; /* notify RDMA-write-to buffer occupation */ - uint8_t tail; /* last word of payload */ -} MPID_nem_dcfa_lmt_cookie_t; - -typedef enum MPID_nem_dcfa_pkt_subtype { - MPIDI_NEM_DCFA_PKT_EAGER_SEND, - MPIDI_NEM_DCFA_PKT_PUT, - MPIDI_NEM_DCFA_PKT_ACCUMULATE, - MPIDI_NEM_DCFA_PKT_GET, - MPIDI_NEM_DCFA_PKT_GET_RESP, - MPIDI_NEM_DCFA_PKT_LMT_GET_DONE, - MPIDI_NEM_DCFA_REQ_SEQ_NUM, - MPIDI_NEM_DCFA_REPLY_SEQ_NUM, - MPIDI_NEM_DCFA_CHG_RDMABUF_OCC_NOTIFY_STATE, - MPIDI_NEM_DCFA_NUM_PKT_HANDLERS -} MPID_nem_dcfa_pkt_subtype_t; - -/* derived from MPID_nem_pkt_netmod_t */ -typedef struct MPID_nem_dcfa_pkt_prefix { - MPID_nem_pkt_type_t type; - unsigned subtype; - /* additional field */ - int seq_num_tail; -} MPID_nem_dcfa_pkt_prefix_t; - -/* derived from MPID_nem_pkt_netmod_t and MPID_nem_pkt_lmt_done_t */ -typedef struct MPID_nem_dcfa_pkt_lmt_get_done { - MPID_nem_pkt_type_t type; - unsigned subtype; - /* additional field */ - MPI_Request req_id; - int seq_num_tail; -} MPID_nem_dcfa_pkt_lmt_get_done_t; - -/* derived from MPID_nem_pkt_netmod_t */ -typedef struct MPID_nem_dcfa_pkt_req_seq_num_t { - MPID_nem_pkt_type_t type; - unsigned subtype; - /* additional field */ - int seq_num_tail; -} MPID_nem_dcfa_pkt_req_seq_num_t; - -/* derived from MPID_nem_pkt_netmod_t */ -typedef struct MPID_nem_dcfa_pkt_reply_seq_num_t { - MPID_nem_pkt_type_t type; - unsigned subtype; - /* additional field */ - int seq_num_tail; -} MPID_nem_dcfa_pkt_reply_seq_num_t; - -/* derived from MPID_nem_pkt_netmod_t */ -typedef struct MPID_nem_dcfa_pkt_change_rdmabuf_occupancy_notify_state_t { - MPID_nem_pkt_type_t type; - unsigned subtype; - /* additional field */ - int state; -} MPID_nem_dcfa_pkt_change_rdmabuf_occupancy_notify_state_t; - -int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_dcfa_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_dcfa_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_dcfa_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_dcfa_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen /* out */ , - MPID_Request ** rreqp /* out */); -int MPID_nem_dcfa_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); -int MPID_nem_dcfa_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); -int MPID_nem_dcfa_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); -int MPID_nem_dcfa_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); -int MPID_nem_dcfa_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc, - MPIDI_CH3_Pkt_t * pkt, - MPIDI_msg_sz_t * buflen, - MPID_Request ** rreqp); - -/* MPID_nem_dcfa_PktHandler_lmt_done is a wrapper of pkt_DONE_handler and calls it */ -/* pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) is not exported */ -int pkt_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, - MPID_Request ** rreqp); - - -#define MPID_nem_dcfa_send_req_seq_num(vc) do { \ - MPID_PKT_DECL_CAST(_upkt, MPID_nem_dcfa_pkt_req_seq_num_t, _pkt); \ - MPID_Request *_req; \ - \ - MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending req_seq_num packet"); \ - MPIDI_Pkt_init(_pkt, MPIDI_NEM_PKT_NETMOD); \ - _pkt->subtype = MPIDI_NEM_DCFA_REQ_SEQ_NUM; \ - \ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); \ - _pkt->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; \ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = vc_dcfa->ibcom->rsr_seq_num_tail; \ - \ - mpi_errno = MPIDI_CH3_iStartMsg((vc), _pkt, sizeof(*_pkt), &_req); \ - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_send_req_seq_num"); \ - if (_req != NULL) { \ - MPIU_ERR_CHKANDJUMP(_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_send_req_seq_num"); \ - MPID_Request_release(_req); \ - } \ - } while (0) - -#define MPID_nem_dcfa_send_reply_seq_num(vc) do { \ - MPID_PKT_DECL_CAST(_upkt, MPID_nem_dcfa_pkt_reply_seq_num_t, _pkt); \ - MPID_Request *_req; \ - \ - MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending reply_seq_num packet"); \ - MPIDI_Pkt_init(_pkt, MPIDI_NEM_PKT_NETMOD); \ - _pkt->subtype = MPIDI_NEM_DCFA_REPLY_SEQ_NUM; \ - \ - int *rsr_seq_num_tail; \ - ibcom_errno = ibcom_rsr_seq_num_tail_get(VC_FIELD(vc, sc->fd), &rsr_seq_num_tail); \ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rsr_seq_num_tail_get"); \ - _pkt->seq_num_tail = *rsr_seq_num_tail; \ - \ - int *rsr_seq_num_tail_last_sent; \ - ibcom_errno = ibcom_rsr_seq_num_tail_last_sent_get(VC_FIELD(vc, sc->fd), &rsr_seq_num_tail_last_sent); \ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rsr_seq_num_tail_last_sent_get"); \ - *rsr_seq_num_tail_last_sent = *rsr_seq_num_tail; \ -\ - mpi_errno = MPIDI_CH3_iStartMsg((vc), _pkt, sizeof(*_pkt), &_req); \ - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_send_reply_seq_num"); \ - if (_req != NULL) { \ - MPIU_ERR_CHKANDJUMP(_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_send_reply_seq_num"); \ - MPID_Request_release(_req); \ - } \ - } while (0) - -#define MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state(vc, _state) do { \ - MPID_PKT_DECL_CAST(_upkt, MPID_nem_dcfa_pkt_change_rdmabuf_occupancy_notify_state_t, _pkt); \ - MPID_Request *_req; \ - \ - MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending change_rdmabuf_occupancy_notify_state packet"); \ - MPIDI_Pkt_init(_pkt, MPIDI_NEM_PKT_NETMOD); \ - _pkt->subtype = MPIDI_NEM_DCFA_CHG_RDMABUF_OCC_NOTIFY_STATE; \ - _pkt->state = _state; \ - \ - mpi_errno = MPIDI_CH3_iStartMsg((vc), _pkt, sizeof(*_pkt), &_req); \ - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state"); \ - if (_req != NULL) { \ - MPIU_ERR_CHKANDJUMP(_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state"); \ - MPID_Request_release(_req); \ - } \ - } while (0) - -#define MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, lsr_seq_num_tail) \ - do { \ - int lsr_seq_num_head; \ - /* sequence number of (largest) in-flight send command */ \ - ibcom_errno = ibcom_sseq_num_get(vc_dcfa->sc->fd, &lsr_seq_num_head); \ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_sseq_num_get"); \ - \ - int *rdmabuf_occupancy_notify_rstate; \ - ibcom_errno = ibcom_rdmabuf_occupancy_notify_rstate_get(vc_dcfa->sc->fd, &rdmabuf_occupancy_notify_rstate); \ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rdmabuf_occupancy_notify_rstate_get"); \ - \ - /*dprintf("notify_policy_lw,head=%d,tail=%d,lw=%d\n", lsr_seq_num_head, *lsr_seq_num_tail, IBCOM_RDMABUF_LOW_WATER_MARK);*/ \ - /* if the number of occupied slot of RDMA-write-to buffer have got below the low water-mark */ \ - if (*rdmabuf_occupancy_notify_rstate == IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW && \ - MPID_nem_dcfa_diff32(lsr_seq_num_head, *lsr_seq_num_tail) < IBCOM_RDMABUF_LOW_WATER_MARK) { \ - dprintf("changing notify_rstate\n"); \ - /* remember remote notifying policy so that local can know when to change remote policy back to HW */ \ - *rdmabuf_occupancy_notify_rstate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW; \ - /* change remote notifying policy of RDMA-write-to buf occupancy info */ \ - MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state(vc, IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW); \ - } \ - } while (0) - -#define MPID_nem_dcfa_lmt_send_GET_DONE(vc, rreq) do { \ - MPID_PKT_DECL_CAST(_upkt, MPID_nem_dcfa_pkt_lmt_get_done_t, _done_pkt); \ - MPID_Request *_done_req; \ - \ - MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending rndv DONE packet"); \ - MPIDI_Pkt_init(_done_pkt, MPIDI_NEM_PKT_NETMOD); \ - _done_pkt->subtype = MPIDI_NEM_DCFA_PKT_LMT_GET_DONE;\ - _done_pkt->req_id = (rreq)->ch.lmt_req_id; \ - /* embed SR occupancy information */ \ - _done_pkt->seq_num_tail = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ - \ - /* remember the last one sent */ \ - VC_FIELD(vc, ibcom->rsr_seq_num_tail_last_sent) = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ - \ - mpi_errno = MPIDI_CH3_iStartMsg((vc), _done_pkt, sizeof(*_done_pkt), &_done_req); \ - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**donepkt"); \ - if (_done_req != NULL) \ - { \ - MPIU_ERR_CHKANDJUMP(_done_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**donepkt"); \ - MPID_Request_release(_done_req); \ - } \ - } while (0) - -#define DCFA_MAX(a, b) ((a) > (b) ? (a) : (b)) - -static inline void *MPID_nem_dcfa_stmalloc(size_t _sz) -{ - size_t sz = _sz; - int i = 0; - do { - i++; - sz >>= 1; - } while (sz > 0); - if (i < 12) { - return malloc(sz); - } - if (i > 30) { - return mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - } - int ndx = i - 12; - void *slot; - if (MPID_nem_dcfa_fl[ndx]) { - slot = MPID_nem_dcfa_fl[ndx]; - if (MPID_nem_dcfa_myrank == 1) { - //printf("stmalloc,reuse %p,%08x\n", slot, (int)_sz); - } - MPID_nem_dcfa_fl[ndx] = *((void **) MPID_nem_dcfa_fl[ndx]); - } - else { - slot = mmap(0, 1 << i, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (MPID_nem_dcfa_myrank == 1) { - //printf("stmalloc,new %p,%08x\n", slot, (int)_sz); - } - } - return slot; -} - -static inline void MPID_nem_dcfa_stfree(void *ptr, size_t sz) -{ - if (MPID_nem_dcfa_myrank == 1) { - //printf("stfree,%p,%08x\n", ptr, (int)sz); - } - int i = 0; - do { - i++; - sz >>= 1; - } while (sz > 0); - if (i < 12) { - free(ptr); - goto fn_exit; - } - if (i > 30) { - munmap(ptr, sz); - goto fn_exit; - } - int ndx = i - 12; - *((void **) ptr) = MPID_nem_dcfa_fl[ndx]; - MPID_nem_dcfa_fl[ndx] = ptr; - fn_exit:; -} -#endif /* DCFA_IMPL_H */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c deleted file mode 100644 index 871feea..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c +++ /dev/null @@ -1,1112 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2006 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - * - * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - */ - -#include "dcfa_impl.h" -#ifdef USE_PMI2_API -#include "pmi2.h" -#else -#include "pmi.h" -#endif - -//#define DEBUG_DCFA_INIT -#ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ -#undef dprintf -#endif -#ifdef DEBUG_DCFA_INIT -#define dprintf printf -#else -#define dprintf(...) -#endif - -MPID_nem_netmod_funcs_t MPIDI_nem_dcfa_funcs = { - MPID_nem_dcfa_init, - MPID_nem_dcfa_finalize, - MPID_nem_dcfa_poll, - MPID_nem_dcfa_get_business_card, - MPID_nem_dcfa_connect_to_root, - MPID_nem_dcfa_vc_init, - MPID_nem_dcfa_vc_destroy, - MPID_nem_dcfa_vc_terminate, - NULL, /*MPID_nem_dcfa_anysource_iprobe */ - NULL, /*MPID_nem_dcfa_anysource_improbe */ -}; - -MPIDI_CH3_PktHandler_Fcn *MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_NUM_PKT_HANDLERS]; - -static MPIDI_Comm_ops_t comm_ops = { - /*NULL, */ MPID_nem_dcfa_recv_posted, - /* recv_posted */ - - NULL, /* send */ - NULL, /* rsend */ - NULL, /* ssend */ - NULL, /* isend */ - NULL, /* irsend */ - NULL, /* issend */ - - NULL, /* send_init */ - NULL, /* bsend_init */ - NULL, /* rsend_init */ - NULL, /* ssend_init */ - NULL, /* startall */ - - NULL, /* cancel_send */ - NULL, /* cancel_recv */ - - NULL, /* probe */ - NULL, /* iprobe */ - NULL, /* improbe */ -}; - -void *MPID_nem_dcfa_fl[18]; -int MPID_nem_dcfa_nranks; -dcfa_conn_ud_t *MPID_nem_dcfa_conn_ud; -dcfaconn_t *MPID_nem_dcfa_conns; -MPIDI_VC_t **MPID_nem_dcfa_pollingset; -int MPID_nem_dcfa_conn_ud_fd; -IbCom *MPID_nem_dcfa_conn_ud_ibcom; -int MPID_nem_dcfa_npollingset; -int *MPID_nem_dcfa_scratch_pad_fds; -//char *MPID_nem_dcfa_recv_buf; -int MPID_nem_dcfa_myrank; -uint64_t MPID_nem_dcfa_tsc_poll; -int MPID_nem_dcfa_ncqe; -int MPID_nem_dcfa_ncqe_lmt_put; -#ifdef DCFA_ONDEMAND -MPID_nem_dcfa_cm_map_t MPID_nem_dcfa_cm_state; -int MPID_nem_dcfa_ncqe_connect; -#endif -int MPID_nem_dcfa_ncqe_scratch_pad; -int MPID_nem_dcfa_ncqe_to_drain; -int MPID_nem_dcfa_ncqe_nces; -MPID_nem_dcfa_lmtq_t MPID_nem_dcfa_lmtq = { NULL, NULL }; -MPID_nem_dcfa_lmtq_t MPID_nem_dcfa_lmt_orderq = { NULL, NULL }; -uint8_t MPID_nem_dcfa_lmt_tail_addr_cbf[MPID_nem_dcfa_cbf_nslot * MPID_nem_dcfa_cbf_bitsperslot / - 8] = { 0 }; -static uint32_t MPID_nem_dcfa_rand_next = 1; -MPID_nem_dcfa_vc_area *MPID_nem_dcfa_debug_current_vc_dcfa; -static int listen_fd; -static int listen_port; - -uint8_t MPID_nem_dcfa_rand() -{ - //return 0xaa; - MPID_nem_dcfa_rand_next = MPID_nem_dcfa_rand_next * 1103515245 + 12345; - return (MPID_nem_dcfa_rand_next / 65536) % 256; -} - -uint64_t MPID_nem_dcfa_rdtsc() -{ - uint64_t x; - __asm__ __volatile__("rdtsc; shl $32, %%rdx; or %%rdx, %%rax":"=a"(x)::"%rdx", "memory"); /* rdtsc cannot be executed earlier than here */ - return x; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_kvs_put_binary -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPID_nem_dcfa_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, int length) -{ - int mpi_errno = MPI_SUCCESS; - int pmi_errno; - char *kvs_name; - char key[256], val[256], str[256]; - int j; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_KVS_PUT_BINARY); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_KVS_PUT_BINARY); - - mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPIDI_PG_GetConnKVSname"); - dprintf("kvs_put_binary,kvs_name=%s\n", kvs_name); - - sprintf(key, "bc/%d/%s", from, postfix); - val[0] = 0; - for (j = 0; j < length; j++) { - sprintf(str, "%02x", buf[j]); - strcat(val, str); - } - dprintf("kvs_put_binary,rank=%d,from=%d,PMI_KVS_Put(%s, %s, %s)\n", MPID_nem_dcfa_myrank, from, - kvs_name, key, val); - pmi_errno = PMI_KVS_Put(kvs_name, key, val); - MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMI_KVS_Put"); - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_KVS_PUT_BINARY); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_kvs_get_binary -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPID_nem_dcfa_kvs_get_binary(int from, const char *postfix, char *buf, int length) -{ - int mpi_errno = MPI_SUCCESS; - int pmi_errno; - char *kvs_name; - char key[256], val[256], str[256]; - int j; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_KVS_GET_BINARY); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_KVS_GET_BINARY); - - mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); - dprintf("kvs_get_binary,kvs_name=%s\n", kvs_name); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPIDI_PG_GetConnKVSname"); - - sprintf(key, "bc/%d/%s", from, postfix); - pmi_errno = PMI_KVS_Get(kvs_name, key, val, 256); - dprintf("kvs_put_binary,rank=%d,from=%d,PMI_KVS_Get(%s, %s, %s)\n", MPID_nem_dcfa_myrank, from, - kvs_name, key, val); - MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMS_KVS_Get"); - - dprintf("rank=%d,obtained val=%s\n", MPID_nem_dcfa_myrank, val); - char *strp = val; - for (j = 0; j < length; j++) { - memcpy(str, strp, 2); - str[2] = 0; - buf[j] = strtol(str, NULL, 16); - strp += 2; - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_KVS_GET_BINARY); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -static int MPID_nem_dcfa_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p); - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_init -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p) -{ - int mpi_errno = MPI_SUCCESS; - int ibcom_errno = 0, pmi_errno; - int ret; - int i, j, k; - int ib_port = 1; - - MPIU_CHKPMEM_DECL(7); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_INIT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_INIT); - - /* first make sure that our private fields in the vc fit into the area provided */ - MPIU_Assert(sizeof(MPID_nem_dcfa_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN); - - MPID_nem_dcfa_nranks = pg_p->size; - MPID_nem_dcfa_myrank = pg_rank; - MPID_nem_dcfa_tsc_poll = MPID_nem_dcfa_rdtsc(); - MPID_nem_dcfa_ncqe = 0; - MPID_nem_dcfa_ncqe_lmt_put = 0; -#ifdef DCFA_ONDEMAND - MPID_nem_dcfa_ncqe_connect = 0; -#endif - MPID_nem_dcfa_ncqe_scratch_pad = 0; - MPID_nem_dcfa_ncqe_to_drain = 0; - MPID_nem_dcfa_ncqe_nces = 0; - MPID_nem_dcfa_npollingset = 0; - -#ifdef DCFA_ONDEMAND - /* prepare UD QPN for dynamic connection */ - ibcom_errno = ibcomOpen(ib_port, IBCOM_OPEN_UD, &MPID_nem_dcfa_conn_ud_fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcomOpen"); - ibcom_errno = ibcom_obtain_pointer(MPID_nem_dcfa_conn_ud_fd, &MPID_nem_dcfa_conn_ud_ibcom); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); - ibcom_errno = ibcom_rts(MPID_nem_dcfa_conn_ud_fd, 0, 0, 0); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rts"); - - for (i = 0; i < IBCOM_MAX_RQ_CAPACITY; i++) { - ibcom_errno = ibcom_udrecv(MPID_nem_dcfa_conn_ud_fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_udrecv"); - } - - /* obtain gid, lid, qpn using KVS */ - MPIU_CHKPMEM_MALLOC(MPID_nem_dcfa_conn_ud, dcfa_conn_ud_t *, - MPID_nem_dcfa_nranks * sizeof(dcfa_conn_ud_t), mpi_errno, - "ud connection table"); - memset(MPID_nem_dcfa_conn_ud, 0, MPID_nem_dcfa_nranks * sizeof(dcfa_conn_ud_t)); - - /* put bc/<my rank>/dcs/gid:lid:qpn */ - uint32_t my_qpnum; - uint16_t my_lid; - union ibv_gid my_gid; - ibcom_get_info_conn(MPID_nem_dcfa_conn_ud_fd, IBCOM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - ibcom_get_info_conn(MPID_nem_dcfa_conn_ud_fd, IBCOM_INFOKEY_PORT_LID, &my_lid, - sizeof(uint16_t)); - ibcom_get_info_conn(MPID_nem_dcfa_conn_ud_fd, IBCOM_INFOKEY_PORT_GID, &my_gid, - sizeof(union ibv_gid)); - - char *kvs_name; - mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); - char *key_dcs, val[2 * sizeof(union ibv_gid) + 1 + 4 + 1 + 8 + 1], str[9]; - - /* count maximum length of the string representation of remote_rank */ - for (i = 0, nranks = MPID_nem_dcfa_nranks; nranks > 0; nranks /= 10, i++) { - } - MPIU_CHKPMEM_MALLOC(key_dcs, char *, strlen("bc/") + i + strlen("/dcs/gid_lid_qpn") + 1, - mpi_errno, "connection table"); - - sprintf(key, "bc/%d/dcs/gid_lid_qpn", MPID_nem_dcfa_myrank); - val[0] = 0; - for (j = 0; j < sizeof(union ibv_gid); j++) { - sprintf(str, "%02x", my_gid.raw[j]); - strcat(val, str); - } - sprintf(str, ":"); - strcat(val, str); - sprintf(str, "%04x:", my_lid); - strcat(val, str); - sprintf(str, "%08x", my_qpnum); - strcat(val, str); - dprintf("rank=%d,PMI_KVS_Put(%s, %s, %s)\n", MPID_nem_dcfa_myrank, kvs_name, key_dcs, val); - pmi_errno = PMI_KVS_Put(kvs_name, key_dcs, val); - MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMI_KVS_Put"); - - /* wait for key-value to propagate among all ranks */ - pmi_errno = PMI_Barrier(); - MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); - - /* obtain GID, LID, QP number for remote UD QP for dynamic connection */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - if (i != MPID_nem_dcfa_myrank) { - sprintf(key_dcs, "bc/%d/dcs/gid_lid_qpn", i); - pmi_errno = PMI_KVS_Get(kvs_name, key_dcs, val, 256); - dprintf("pmi_errno=%d\n", pmi_errno); - MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMI_KVS_Get"); - dprintf("rank=%d,obtained val=%s\n", MPID_nem_dcfa_myrank, val); - char *strp = val; - for (j = 0; j < sizeof(union ibv_gid); j++) { - memcpy(str, strp, 2); - str[2] = 0; - MPID_nem_dcfa_conn_ud[i].gid.raw[j] = strtol(str, NULL, 16); - strp += 2; - } - sscanf(strp, ":%04x:%08x", &MPID_nem_dcfa_conn_ud[i].lid, - &MPID_nem_dcfa_conn_ud[i].qpn); - - dprintf("remote rank=%d,gid=", i); - for (j = 0; j < sizeof(union ibv_gid); j++) { - dprintf("%02x", MPID_nem_dcfa_conn_ud[i].gid.raw[j]); - } - dprintf(",lid=%04x,qpn=%08x\n", MPID_nem_dcfa_conn_ud[i].lid, - MPID_nem_dcfa_conn_ud[i].qpn); - } - } -#endif - - /* malloc scratch-pad fd */ - MPIU_CHKPMEM_MALLOC(MPID_nem_dcfa_scratch_pad_fds, int *, MPID_nem_dcfa_nranks * sizeof(int), - mpi_errno, "connection table"); - memset(MPID_nem_dcfa_scratch_pad_fds, 0, MPID_nem_dcfa_nranks * sizeof(int)); - - /* prepare scrath-pad QP and malloc scratch-pad */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - ibcom_errno = ibcomOpen(ib_port, IBCOM_OPEN_SCRATCH_PAD, &MPID_nem_dcfa_scratch_pad_fds[i]); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcomOpen"); - - ibcom_errno = - ibcom_alloc(MPID_nem_dcfa_scratch_pad_fds[i], - MPID_nem_dcfa_nranks * sizeof(ibcom_qp_state_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_alloc"); - } - - /* put bc/me/sp/{gid,lid} put bc/me/sp/{qpn,rmem,rkey}/you */ - int nranks; - -#ifndef DCFA_ONDEMAND - uint32_t my_qpnum; - uint16_t my_lid; - union ibv_gid my_gid; -#endif - void *my_rmem; - int my_rkey; - - int remote_qpnum; - uint16_t remote_lid; - union ibv_gid remote_gid; - void *remote_rmem; - int remote_rkey; - - char *remote_rank_str; - char *key_str; - - /* count maximum length of the string representation of remote_rank */ - for (i = 0, nranks = MPID_nem_dcfa_nranks; nranks > 0; nranks /= 10, i++) { - } - MPIU_CHKPMEM_MALLOC(remote_rank_str, char *, 1 + i + 1, mpi_errno, "connection table"); - MPIU_CHKPMEM_MALLOC(key_str, char *, strlen("sp/qpn") + 1 + i + 1, mpi_errno, - "connection table"); - - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - - if (i == 0) { - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_scratch_pad_fds[i], IBCOM_INFOKEY_PORT_LID, - &my_lid, sizeof(uint16_t)); - dprintf("dcfa_init,scratch pad,lid=%04x\n", my_lid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, "sp/lid", (uint8_t *) & my_lid, - sizeof(uint16_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_kvs_put_binary"); - - { - dprintf("dcfa_init,scratch pad,put <%d/sp/lid/,%04x>\n", MPID_nem_dcfa_myrank, - (int) my_lid); - } - - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_scratch_pad_fds[i], IBCOM_INFOKEY_PORT_GID, - &my_gid, sizeof(union ibv_gid)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, "sp/gid", (uint8_t *) & my_gid, - sizeof(union ibv_gid)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_kvs_put_binary"); - - dprintf("dcfa_init,scratch pad,gid "); - for (k = 0; k < 16; k++) { - dprintf("%02x", (int) my_gid.raw[k]); - } - dprintf("\n"); - } - - /* put bc/me/sp/qpn/you */ - strcpy(key_str, "sp/qpn"); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_scratch_pad_fds[i], IBCOM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - dprintf("dcfa_init,scratch pad,qpn=%08x\n", my_qpnum); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - dprintf("dcfa_init,scratch pad,kvs put done\n"); - - strcpy(key_str, "sp/rmem"); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - - ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_scratch_pad_fds[i], IBCOM_SCRATCH_PAD_TO, - IBCOM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); - - dprintf("dcfa_init,scratch_pad,rmem=%p\n", my_rmem); - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_rmem, - sizeof(void *)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - - strcpy(key_str, "sp/rkey"); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - - ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_scratch_pad_fds[i], IBCOM_SCRATCH_PAD_TO, - IBCOM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); - dprintf("dcfa_init,scratch_pad,rkey=%08x\n", my_rkey); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_rkey, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - } - - /* wait until key-value propagates among all ranks */ - pmi_errno = PMI_Barrier(); - MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); - dprintf("dcfa_init,put KVS;barrier;\n"); - - /* make me-to-you scratch-pad QP RTS */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - if (i != MPID_nem_dcfa_myrank) { - - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, "sp/gid", (char *) &remote_gid, - sizeof(union ibv_gid)); - dprintf("dcfa_init,after kvs get\n"); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, "sp/lid", (char *) &remote_lid, sizeof(uint16_t)); - dprintf("dcfa_init,after kvs get\n"); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - strcpy(key_str, "sp/qpn"); - strcat(key_str, ""); /* "" or "lmt-put" */ - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_qpnum, sizeof(uint32_t)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - dprintf("dcfa_init,get KVS,remote_qpnum=%08x\n", remote_qpnum); - - ibcom_errno = - ibcom_rts(MPID_nem_dcfa_scratch_pad_fds[i], remote_qpnum, remote_lid, &remote_gid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rts"); - - strcpy(key_str, "sp/rmem"); - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_rmem, sizeof(void *)); - dprintf("dcfa_init,after kvs get\n"); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - dprintf("dcfa_init,get KVS,remote_rmem=%p\n", remote_rmem); - - strcpy(key_str, "sp/rkey"); - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_rkey, sizeof(uint32_t)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - dprintf("dcfa_init,get KVS,remote_rkey=%08x\n", remote_rkey); - - ibcom_errno = - ibcom_reg_mr_connect(MPID_nem_dcfa_scratch_pad_fds[i], remote_rmem, remote_rkey); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_reg_mr_connect"); - } - } - - /* wait until you-to-me scratch-pad QP becomes RTR */ - pmi_errno = PMI_Barrier(); - MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); - - MPIU_CHKPMEM_MALLOC(MPID_nem_dcfa_conns, dcfaconn_t *, - MPID_nem_dcfa_nranks * sizeof(dcfaconn_t), mpi_errno, "connection table"); - memset(MPID_nem_dcfa_conns, 0, MPID_nem_dcfa_nranks * sizeof(dcfaconn_t)); - - MPIU_CHKPMEM_MALLOC(MPID_nem_dcfa_pollingset, MPIDI_VC_t **, - MPID_NEM_DCFA_MAX_POLLINGSET * sizeof(MPIDI_VC_t *), mpi_errno, - "connection table"); - memset(MPID_nem_dcfa_pollingset, 0, MPID_NEM_DCFA_MAX_POLLINGSET * sizeof(MPIDI_VC_t *)); - - /* prepare eager-send QP */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - ibcom_errno = ibcomOpen(ib_port, IBCOM_OPEN_RC, &MPID_nem_dcfa_conns[i].fd); - dprintf("init,fd=%d\n", MPID_nem_dcfa_conns[i].fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcomOpen"); - } - -#if 0 - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - ibcom_errno = ibcomOpen(ib_port, IBCOM_OPEN_RC_LMT_PUT, &MPID_nem_dcfa_conns[i].fd_lmt_put); - dprintf("init,fd_lmt_put=%d\n", MPID_nem_dcfa_conns[i].fd_lmt_put); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcomOpen"); - } -#endif - - /* put bc/me/{gid,lid}, put bc/me/{qpn,rmem,rkey}/you */ - mpi_errno = MPID_nem_dcfa_announce_network_addr(pg_rank, bc_val_p, val_max_sz_p); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - /* wait until key-value propagates among all ranks */ - pmi_errno = PMI_Barrier(); - MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); - - /* make me-to-you eager-send QP RTS */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - if (i != MPID_nem_dcfa_myrank) { - - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, MPIDI_CH3I_LID_KEY, (char *) &remote_lid, - sizeof(uint16_t)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, MPIDI_CH3I_GID_KEY, (char *) &remote_gid, - sizeof(union ibv_gid)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - strcpy(key_str, MPIDI_CH3I_RMEM_KEY); - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_rmem, sizeof(void *)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - strcpy(key_str, MPIDI_CH3I_RKEY_KEY); - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_rkey, sizeof(uint32_t)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - strcpy(key_str, MPIDI_CH3I_QPN_KEY); - strcat(key_str, ""); /* "" or "lmt-put" */ - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_qpnum, sizeof(uint32_t)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - dprintf("remote_qpnum obtained=%08x\n", remote_qpnum); - - ibcom_errno = - ibcom_rts(MPID_nem_dcfa_conns[i].fd, remote_qpnum, remote_lid, &remote_gid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rts"); - - /* report me-to-you eager-send QP becomes RTR */ - IbCom *ibcom_scratch_pad; - ibcom_errno = - ibcom_obtain_pointer(MPID_nem_dcfa_scratch_pad_fds[i], &ibcom_scratch_pad); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); - - ibcom_qp_state_t state = {.state = IBCOM_QP_STATE_RTR }; - ibcom_errno = - ibcom_put_scratch_pad(MPID_nem_dcfa_scratch_pad_fds[i], - (uint64_t) ibcom_scratch_pad, - sizeof(ibcom_qp_state_t) * MPID_nem_dcfa_myrank, - sizeof(ibcom_qp_state_t), (void *) &state); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_put_scratch_pad"); - MPID_nem_dcfa_ncqe_scratch_pad += 1; - - ibcom_errno = ibcom_reg_mr_connect(MPID_nem_dcfa_conns[i].fd, remote_rmem, remote_rkey); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_reg_mr_connect"); - dprintf("dcfa_init,after mr_connect for me-to-you eager-send QP\n"); - -#if 0 - /* CQ, SQ, SCQ for lmt-put */ - strcpy(key_str, MPIDI_CH3I_QPN_KEY); - strcat(key_str, "lmt-put"); /* "" or "lmt-put" */ - sprintf(remote_rank_str, "/%x", MPID_nem_dcfa_myrank); - strcat(key_str, remote_rank_str); - mpi_errno = - MPID_nem_dcfa_kvs_get_binary(i, key_str, (char *) &remote_qpnum, sizeof(uint32_t)); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - ibcom_errno = - ibcom_rts(MPID_nem_dcfa_conns[i].fd_lmt_put, remote_qpnum, remote_lid, &remote_gid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rts"); -#endif - } - } - -#if 0 /* debug */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - dprintf("init,fd[%d]=%d\n", i, MPID_nem_dcfa_conns[i].fd); - } -#endif - - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_INIT); - return mpi_errno; - fn_fail: - MPIU_CHKPMEM_REAP(); - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_get_business_card -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p) -{ - int mpi_errno = MPI_SUCCESS; - int str_errno = MPIU_STR_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_GET_BUSINESS_CARD); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_GET_BUSINESS_CARD); - dprintf("MPID_nem_dcfa_get_business_card,enter\n"); - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_GET_BUSINESS_CARD); - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_announce_network_addr -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPID_nem_dcfa_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p) -{ - int mpi_errno = MPI_SUCCESS; - int str_errno = MPIU_STR_SUCCESS; - int ibcom_errno; - int i, j, nranks; - - uint32_t my_qpnum; - uint16_t my_lid; - union ibv_gid my_gid; - void *my_rmem; - int my_rkey; - char *remote_rank_str; /* perl -e '$key_str .= $remote_rank;' */ - char *key_str; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_ANNOUNCE_NETWORK_ADDR); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_ANNOUNCE_NETWORK_ADDR); - MPIU_CHKLMEM_DECL(2); /* argument is the number of alloca */ - - /* count maximum length of the string representation of remote_rank */ - for (i = 0, nranks = MPID_nem_dcfa_nranks; nranks > 0; nranks /= 10, i++) { - } - MPIU_CHKLMEM_MALLOC(remote_rank_str, char *, i + 1, mpi_errno, "key_str"); /* alloca */ - MPIU_CHKLMEM_MALLOC(key_str, char *, strlen(MPIDI_CH3I_QPN_KEY) + i + 1, mpi_errno, "key_str"); /* alloca */ - - /* We have one local qp and remote qp for each rank-pair, - * so a rank should perform - * remote_qpn = kvs_get($remote_rank . "qpnum/" . $local_rank). - * a memory area to read from and write to HCA, - * and a memory area to read from HCA and write to DRAM is - * associated with each connection, so a rank should perform - * rkey = kvs_get($remote_rank . "rkey/" . $local_rank) - * and raddr = kvs_get($remote_rank . "raddr/" . $local_rank). */ - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - - /* lid and gid are common for all remote-ranks */ - if (i == 0) { - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[i].fd, IBCOM_INFOKEY_PORT_LID, &my_lid, - sizeof(uint16_t)); - dprintf("get_business_card,lid=%04x\n", my_lid); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, MPIDI_CH3I_LID_KEY, - (uint8_t *) & my_lid, sizeof(uint16_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_kvs_put_binary"); - - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[i].fd, IBCOM_INFOKEY_PORT_GID, &my_gid, - sizeof(union ibv_gid)); - - dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); - dprintf("get_business_card,sz=%ld,my_gid=", sizeof(union ibv_gid)); - for (j = 0; j < 16; j++) { - dprintf("%02x", (int) my_gid.raw[j]); - } - dprintf("\n"); - - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, MPIDI_CH3I_GID_KEY, - (uint8_t *) & my_gid, sizeof(union ibv_gid)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_kvs_put_binary"); - dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); - } - - /* we use different RDMA-rbuf for different senders. - * so announce like this: - * <"0/qpn/0", 0xa0000> - * <"0/qpn/1", 0xb0000> - * <"0/qpn/2", 0xc0000> - * <"0/qpn/3", 0xd0000> - */ - strcpy(key_str, MPIDI_CH3I_QPN_KEY); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[i].fd, IBCOM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - -#if 0 - /* lmt-put */ - strcpy(key_str, MPIDI_CH3I_QPN_KEY); - strcat(key_str, "lmt-put"); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[i].fd_lmt_put, IBCOM_INFOKEY_QP_QPN, &my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_qpnum, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); -#endif - - strcpy(key_str, MPIDI_CH3I_RMEM_KEY); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - - ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_conns[i].fd, IBCOM_RDMAWR_TO, IBCOM_INFOKEY_MR_ADDR, - &my_rmem, sizeof(void *)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); - - dprintf("rmem=%p\n", my_rmem); - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_rmem, - sizeof(void *)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - - strcpy(key_str, MPIDI_CH3I_RKEY_KEY); - sprintf(remote_rank_str, "/%x", i); - strcat(key_str, remote_rank_str); - - ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_conns[i].fd, IBCOM_RDMAWR_TO, IBCOM_INFOKEY_MR_RKEY, - &my_rkey, sizeof(int)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); - - mpi_errno = - MPID_nem_dcfa_kvs_put_binary(MPID_nem_dcfa_myrank, key_str, (uint8_t *) & my_rkey, - sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_ANNOUNCE_NETWORK_ADDR); - return mpi_errno; - fn_fail: - MPIU_CHKLMEM_FREEALL(); - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_connect_to_root -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc) -{ - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_CONNECT_TO_ROOT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_CONNECT_TO_ROOT); - - dprintf("toroot,%d->%d", MPID_nem_dcfa_myrank, new_vc->pg_rank); - /* not implemented */ - - //fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_CONNECT_TO_ROOT); - return MPI_SUCCESS; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_vc_init -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_vc_init(MPIDI_VC_t * vc) -{ - MPIDI_CH3I_VC *vc_ch = VC_CH(vc); - int mpi_errno = MPI_SUCCESS; - - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - int ibcom_errno; - size_t s; - dcfaconn_t *sc; - off_t offset; - - int remote_qpnum; - uint16_t remote_lid; - union ibv_gid remote_gid; - void *remote_rmem; - int remote_rkey; - - char key_str[256], remote_rank_str[256]; - - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_VC_INIT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_VC_INIT); - - vc_dcfa->sc = &MPID_nem_dcfa_conns[vc->pg_rank]; - - /* store pointer to ibcom */ - ibcom_errno = ibcom_obtain_pointer(MPID_nem_dcfa_conns[vc->pg_rank].fd, &vc_dcfa->ibcom); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); - - ibcom_errno = - ibcom_obtain_pointer(MPID_nem_dcfa_conns[vc->pg_rank].fd_lmt_put, &vc_dcfa->ibcom_lmt_put); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); - - //dprintf("vc_init,open,fd=%d,ptr=%p,rsr_seq_num_poll=%d\n", MPID_nem_dcfa_conns[vc->pg_rank].fd, vc_dcfa->ibcom, vc_dcfa->ibcom->rsr_seq_num_poll); - - /* initialize sendq */ - vc_dcfa->sendq.head = NULL; - vc_dcfa->sendq.tail = NULL; - vc_dcfa->sendq_lmt_put.head = NULL; - vc_dcfa->sendq_lmt_put.tail = NULL; - - /* rank is sent as wr_id and used to obtain vc in poll */ - MPID_nem_dcfa_conns[vc->pg_rank].vc = vc; - MPIU_ERR_CHKANDJUMP(MPID_nem_dcfa_npollingset + 1 > MPID_NEM_DCFA_MAX_POLLINGSET, mpi_errno, - MPI_ERR_OTHER, "**MPID_nem_dcfa_npollingset"); - MPID_nem_dcfa_pollingset[MPID_nem_dcfa_npollingset++] = vc; - //printf("vc_init,%d->%d,vc=%p,npollingset=%d\n", MPID_nem_dcfa_myrank, vc->pg_rank, vc, MPID_nem_dcfa_npollingset); - - /* wait until you-to-me eager-send QP becomes RTR */ - IbCom *ibcom_scratch_pad; - ibcom_errno = - ibcom_obtain_pointer(MPID_nem_dcfa_scratch_pad_fds[vc->pg_rank], &ibcom_scratch_pad); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); - - int ntrial = 0; - volatile ibcom_qp_state_t *rstate = - (ibcom_qp_state_t *) ((uint8_t *) ibcom_scratch_pad->icom_mem[IBCOM_SCRATCH_PAD_TO] + - vc->pg_rank * sizeof(ibcom_qp_state_t)); - dprintf("dcfa_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate)); - while (rstate->state != IBCOM_QP_STATE_RTR) { - __asm__ __volatile__("pause;":::"memory"); - if (++ntrial > 1024) { - /* detect RDMA-write failure */ - ibcom_errno = MPID_nem_dcfa_drain_scq_scratch_pad(); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq_scratch_pad"); - } - } - dprintf("dcfa_init,you-to-me eager-send QP is RTR\n"); - - /* post IBCOM_MAX_SQ_CAPACITY of recv commands beforehand, replenish when retiring them in dcfa_poll */ - int i; - for (i = 0; i < IBCOM_MAX_RQ_CAPACITY; i++) { - //dprintf("irecv,%d->%d\n", MPID_nem_dcfa_myrank, vc->pg_rank); - ibcom_errno = ibcom_irecv(MPID_nem_dcfa_conns[vc->pg_rank].fd, (uint64_t) vc->pg_rank); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_irecv"); - } - - MPIDI_CHANGE_VC_STATE(vc, ACTIVE); - - - uint32_t max_msg_sz; - ibcom_get_info_conn(MPID_nem_dcfa_conns[vc->pg_rank].fd, IBCOM_INFOKEY_PATTR_MAX_MSG_SZ, - &max_msg_sz, sizeof(max_msg_sz)); - VC_FIELD(vc, pending_sends) = 0; -#ifdef DCFA_ONDEMAND - VC_FIELD(vc, is_connected) = 0; -#endif - - MPIU_Assert(sizeof(sz_hdrmagic_t) == 8); /* assumption in dcfa_ibcom.h */ - MPIU_Assert(sizeof(tailmagic_t) == 1); /* in dcfa_ibcom.h */ - - uint32_t sz; -#if 0 - /* assumption in released(), must be power of two */ - sz = IBCOM_RDMABUF_NSEG; - while ((sz & 1) == 0) { - sz >>= 1; - } - sz >>= 1; - if (sz) { - MPIU_Assert(0); - } -#endif - - /* assumption in dcfa_poll.c, must be power of two */ - for (sz = IBCOM_RDMABUF_SZSEG; sz > 0; sz >>= 1) { - if (sz != 1 && (sz & 1)) { - MPIU_Assert(0); - } - } - - char *val; - val = getenv("MP2_IBA_EAGER_THRESHOLD"); - vc->eager_max_msg_sz = val ? atoi(val) : MPID_NEM_DCFA_EAGER_MAX_MSG_SZ; - vc->ready_eager_max_msg_sz = val ? atoi(val) : MPID_NEM_DCFA_EAGER_MAX_MSG_SZ; - dprintf("dcfa_vc_init,vc->eager_max_msg_sz=%d\n", vc->eager_max_msg_sz); - - /* vc->rndvSend_fn is set in MPID_nem_vc_init (in src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c) */ - ; - vc->sendNoncontig_fn = MPID_nem_dcfa_SendNoncontig; - - vc->comm_ops = &comm_ops; - - - /* register packet handler */ - vc_ch->pkt_handler = MPID_nem_dcfa_pkt_handler; - vc_ch->num_pkt_handlers = MPIDI_NEM_DCFA_NUM_PKT_HANDLERS; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_PKT_EAGER_SEND] = MPID_nem_dcfa_PktHandler_EagerSend; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_PKT_PUT] = MPID_nem_dcfa_PktHandler_Put; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_PKT_GET] = MPID_nem_dcfa_PktHandler_Get; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_PKT_GET_RESP] = MPID_nem_dcfa_PktHandler_GetResp; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_PKT_ACCUMULATE] = MPID_nem_dcfa_PktHandler_Accumulate; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_PKT_LMT_GET_DONE] = MPID_nem_dcfa_pkt_GET_DONE_handler; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_REQ_SEQ_NUM] = MPID_nem_dcfa_PktHandler_req_seq_num; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_REPLY_SEQ_NUM] = - MPID_nem_dcfa_PktHandler_reply_seq_num; - MPID_nem_dcfa_pkt_handler[MPIDI_NEM_DCFA_CHG_RDMABUF_OCC_NOTIFY_STATE] = - MPID_nem_dcfa_PktHandler_change_rdmabuf_occupancy_notify_state; - - /* register CH3 send/recv functions */ - vc_ch->iStartContigMsg = MPID_nem_dcfa_iStartContigMsg; - vc_ch->iSendContig = MPID_nem_dcfa_iSendContig; - - /* register CH3--lmt send/recv functions */ - vc_ch->lmt_initiate_lmt = MPID_nem_dcfa_lmt_initiate_lmt; - vc_ch->lmt_start_recv = MPID_nem_dcfa_lmt_start_recv; - vc_ch->lmt_handle_cookie = MPID_nem_dcfa_lmt_handle_cookie; - vc_ch->lmt_done_send = MPID_nem_dcfa_lmt_done_send; - vc_ch->lmt_done_recv = MPID_nem_dcfa_lmt_done_recv; - vc_ch->lmt_vc_terminated = MPID_nem_dcfa_lmt_vc_terminated; - vc_ch->next = NULL; - vc_ch->prev = NULL; - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_VC_INIT); - return mpi_errno; - fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_vc_destroy -#undef FCNAME -#define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_vc_destroy(MPIDI_VC_t * vc) -{ - int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_VC_DESTROY); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_VC_DESTROY); - /* currently do nothing */ - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_VC_DESTROY); - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_vc_terminate -#undef FCNAME -#define FCNAME MPIU_QUOTE(FUNCNAME) -int MPID_nem_dcfa_vc_terminate(MPIDI_VC_t * vc) -{ - dprintf("dcfa_vc_terminate,enter\n"); - int mpi_errno = MPI_SUCCESS; - int ibcom_errno; - int req_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_VC_TERMINATE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_VC_TERMINATE); - - /* Check to make sure that it's OK to terminate the - * connection without making sure that all sends have been sent */ - /* it is safe to only check command queue because - * data transactions always proceed after confirming send by MPI_Wait - * and control transactions always proceed after receiveing reply */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - - dprintf("init,before,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", - MPID_nem_dcfa_myrank, vc->pg_rank, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail), - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), MPID_nem_dcfa_ncqe, VC_FIELD(vc, - pending_sends)); - - /* update remote RDMA-write-to buffer occupancy */ -#if 0 /* we can't send it when the other party has closed QP */ - while (MPID_nem_dcfa_diff32 - (vc_dcfa->ibcom->rsr_seq_num_tail, vc_dcfa->ibcom->rsr_seq_num_tail_last_sent) > 0) { - MPID_nem_dcfa_send_reply_seq_num(vc); - } -#endif - - /* update local RDMA-write-to buffer occupancy */ -#if 0 - while (MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail) > 0) { - MPID_nem_dcfa_poll_eager(vc); - } -#endif - - /* drain sendq */ - while (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)) { - MPID_nem_dcfa_send_progress(vc_dcfa); - } - - dprintf("init,middle,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", - MPID_nem_dcfa_myrank, vc->pg_rank, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail), - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), MPID_nem_dcfa_ncqe, VC_FIELD(vc, - pending_sends)); - - if (MPID_nem_dcfa_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { - usleep(1000); - MPID_nem_dcfa_drain_scq(0); - } - dprintf("init,middle2,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", - MPID_nem_dcfa_myrank, vc->pg_rank, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail), - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), MPID_nem_dcfa_ncqe, VC_FIELD(vc, - pending_sends)); - - if (MPID_nem_dcfa_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { - usleep(1000); - MPID_nem_dcfa_drain_scq(0); - } -#if 0 - /* drain scq */ - while (MPID_nem_dcfa_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { - usleep(1000); - MPID_nem_dcfa_drain_scq(0); - //printf("%d\n", VC_FIELD(vc, pending_sends)); - //printf("%d\n", MPID_nem_dcfa_ncqe); - } -#endif - - dprintf("init,after ,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", - MPID_nem_dcfa_myrank, vc->pg_rank, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail), - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), MPID_nem_dcfa_ncqe, VC_FIELD(vc, - pending_sends)); - - /* drain scratch-pad scq */ - ibcom_errno = MPID_nem_dcfa_drain_scq_scratch_pad(); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq_scratch_pad"); - - mpi_errno = MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED); - if (mpi_errno) { - MPIU_ERR_POP(mpi_errno); - } - - fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_VC_TERMINATE); - return mpi_errno; - fn_fail: - MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno)); - goto fn_exit; - -} diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c deleted file mode 100644 index bfd23d9..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c +++ /dev/null @@ -1,321 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2012 NEC Corporation - * Author: Masamichi Takagi - * (C) 2012 Oct 10 Min Si - * (C) 2001-2009 Yutaka Ishikawa - * See COPYRIGHT in top-level directory. - */ - -#include <unistd.h> -#include <stdlib.h> -#include "dcfa_ibcom.h" - -//#define DEBUG_REG_MR -#ifdef DEBUG_REG_MR -#define dprintf printf -#else -#define dprintf(...) -#endif - -/* cache size of ibv_reg_mr */ -#define IBCOM_REG_MR_NLINE 4096 -#define IBCOM_REG_MR_NWAY 1024 - -#define IBCOM_REG_MR_SZPAGE 4096 -#define IBCOM_REG_MR_LOGSZPAGE 12 - -/* arena allocator */ - -#define NIALLOCID 32 -typedef struct { - char *next; -} free_list_t; -static char *free_list_front[NIALLOCID] = { 0 }; -static char *arena_flist[NIALLOCID] = { 0 }; - -#define SZARENA 4096 -#define CLUSTER_SIZE (SZARENA/sz) -#define ROUNDUP64(addr, align) ((addr + align - 1) & ~((unsigned long)align - 1)) -#define NCLUST_SLAB 1 -#define IBCOM_AALLOC_ID_MRCACHE 0 - -static inline void *aalloc(size_t sz, int id) -{ -#if 1 /* debug */ - return malloc(sz); -#else - char *p = free_list_front[id]; - if ((unsigned long) p & (SZARENA - 1)) { - free_list_front[id] += sz; - return p; - } - else { - char *q, r; - if (arena_flist[id]) { - q = arena_flist[id]; - arena_flist[id] = ((free_list_t *) arena_flist[id])->next; - } - else { - q = mmap(NULL, ROUNDUP64(SZARENA * NCLUST_SLAB, 4096), PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -#if NCLUST_SLAB > 1 - arena_flist[id] = q + SZARENA; - for (p = arena_flist[id]; p < q + (NCLUST_SLAB - 1) * SZARENA; p += SZARENA) { - ((free_list_t *) p)->next = p + SZARENA; - } - ((free_list_t *) p)->next = 0; -#endif - } - *((int *) q) = CLUSTER_SIZE - 1; - // dprintf("q=%llx\n", q); - q += sz + (SZARENA % sz); - free_list_front[id] = q + sz; - return q; - } -#endif -} - -static inline void afree(const void *p, int id) -{ -#if 1 /* debug */ - return free((void *) p); -#else - p = (void *) ((unsigned long) p & ~(SZARENA - 1)); - if (!(--(*((int *) p)))) { - ((free_list_t *) p)->next = arena_flist[id]; - arena_flist[id] = (char *) p; - } -#endif -} - -struct ibcom_reg_mr_listnode_t { - struct ibcom_reg_mr_listnode_t *lru_next; - struct ibcom_reg_mr_listnode_t *lru_prev; -}; - -struct ibcom_reg_mr_cache_entry_t { - /* : public ibcom_reg_mr_listnode_t */ - struct ibcom_reg_mr_listnode_t *lru_next; - struct ibcom_reg_mr_listnode_t *lru_prev; - - struct ibv_mr *mr; - void *addr; - int len; - int refc; -}; - -static struct ibcom_reg_mr_listnode_t ibcom_reg_mr_cache[IBCOM_REG_MR_NLINE]; - -__inline__ int ibcom_hash_func(char *addr) -{ - unsigned int v = (unsigned int) (unsigned long) addr; - //v = v >> IBCOM_REG_MR_LOGSZPAGE; /* assume it is page aligned */ - v = v & (IBCOM_REG_MR_NLINE - 1); - return (int) v; -} - -static void ibcom_reg_mr_insert(struct ibcom_reg_mr_listnode_t *c, struct ibcom_reg_mr_listnode_t *e) -{ - struct ibcom_reg_mr_listnode_t *next; - struct ibcom_reg_mr_listnode_t *prev; - prev = c; - next = prev->lru_next; - e->lru_next = next; - e->lru_prev = prev; - next->lru_prev = e; - prev->lru_next = e; -} - -static void ibcom_reg_mr_unlink(struct ibcom_reg_mr_listnode_t *e) -{ - struct ibcom_reg_mr_listnode_t *next, *prev; - next = e->lru_next; - prev = e->lru_prev; - next->lru_prev = prev; - prev->lru_next = next; -} - -static inline void __lru_queue_display() -{ - struct ibcom_reg_mr_cache_entry_t *p; - int i = 0; - for (i = 0; i < IBCOM_REG_MR_NLINE; i++) { - dprintf("---- hash %d\n", i); - for (p = (struct ibcom_reg_mr_cache_entry_t *) ibcom_reg_mr_cache[i].lru_next; - p != (struct ibcom_reg_mr_cache_entry_t *) &ibcom_reg_mr_cache[i]; - p = (struct ibcom_reg_mr_cache_entry_t *) p->lru_next) { - if (p && p->addr) { - dprintf("-------- p=%p,addr=%p,len=%d,refc=%d,lru_next=%p\n", p, p->addr, p->len, - p->refc, p->lru_next); - } - else { - dprintf("-------- p=%p,lru_next=%p\n", p, p->lru_next); - } - } - } -} - -struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) -{ -#if 0 /* debug */ - struct ibv_mr *mr; - int ibcom_errno = ibcom_reg_mr(addr, len, &mr); - printf("mrcache,ibcom_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len, mr->lkey, - mr->rkey); - if (ibcom_errno != 0) { - goto fn_fail; - } - fn_exit: - return mr; - fn_fail: - goto fn_exit; -#else - int ibcom_errno; - int key; - struct ibcom_reg_mr_cache_entry_t *e; - -#if 1 /*def HAVE_LIBDCFA */ - /* we can't change addr because ibv_post_send assumes mr->host_addr (output of this function) - * must have an exact mirror value of addr (input of this function) */ - void *addr_aligned = addr; - int len_aligned = len; -#else - void *addr_aligned = (void *) ((unsigned long) addr & ~(IBCOM_REG_MR_SZPAGE - 1)); - int len_aligned = - ((((unsigned long) addr + len) - (unsigned long) addr_aligned + IBCOM_REG_MR_SZPAGE - - 1) & ~(IBCOM_REG_MR_SZPAGE - 1)); -#endif - key = ibcom_hash_func(addr); - - dprintf("[MrCache] addr=%p, len=%d\n", addr, len); - dprintf("[MrCache] aligned addr=%p, len=%d\n", addr_aligned, len_aligned); - - //__lru_queue_display(); - int way = 0; - for (e = (struct ibcom_reg_mr_cache_entry_t *) ibcom_reg_mr_cache[key].lru_next; - e != (struct ibcom_reg_mr_cache_entry_t *) &ibcom_reg_mr_cache[key]; - e = (struct ibcom_reg_mr_cache_entry_t *) e->lru_next, way++) { - //dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next); - - if (e->addr <= addr_aligned && (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) { - //dprintf - //("ibcom_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n", - //e->addr, e->len, e->mr->addr, e->mr->length, addr, len); - goto hit; - } - } - - // miss - - // evict an entry and de-register its MR when the cache-set is full - if (way > IBCOM_REG_MR_NWAY) { - struct ibcom_reg_mr_cache_entry_t *victim = - (struct ibcom_reg_mr_cache_entry_t *) e->lru_prev; - ibcom_reg_mr_unlink((struct ibcom_reg_mr_listnode_t *) victim); - - //dprintf("ibcom_reg_mr,evict,entry addr=%p,len=%d,mr addr=%p,len=%ld\n", e->addr, e->len, - //e->mr->addr, e->mr->length); - ibcom_errno = ibcom_dereg_mr(victim->mr); - if (ibcom_errno) { - printf("mrcache,ibcom_dereg_mr\n"); - goto fn_fail; - } - afree(victim, IBCOM_AALLOC_ID_MRCACHE); - } - - e = aalloc(sizeof(struct ibcom_reg_mr_cache_entry_t), IBCOM_AALLOC_ID_MRCACHE); - /* reference counter is used when evicting entry */ - e->refc = 1; - - dprintf("ibcom_reg_mr_fetch,miss,addr=%p,len=%d\n", addr_aligned, len_aligned); - /* register memory */ - ibcom_errno = ibcom_reg_mr(addr_aligned, len_aligned, &e->mr); - if (ibcom_errno != 0) { - fprintf(stderr, "mrcache,ibcom_reg_mr\n"); - goto fn_fail; - } - e->addr = addr_aligned; - e->len = len_aligned; - - //dprintf("ibcom_reg_mr_fetch,fill,e=%p,key=%d,mr=%p,mr addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, - //key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); - - /* register to cache */ - ibcom_reg_mr_insert(&ibcom_reg_mr_cache[key], (struct ibcom_reg_mr_listnode_t *) e); - - //__lru_queue_display(); - - goto fn_exit; - - hit: - - /* reference counter is used when evicting entry */ - e->refc++; -#if 0 /* disable for debug */ - /* move to head of the list */ - if (e != (struct ibcom_reg_mr_cache_entry_t *) ibcom_reg_mr_cache[key].lru_next) { - ibcom_reg_mr_unlink((struct ibcom_reg_mr_listnode_t *) e); - ibcom_reg_mr_insert(&ibcom_reg_mr_cache[key], (struct ibcom_reg_mr_listnode_t *) e); - } -#endif - //dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, - //key, e->mr, e->refc, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); - - //__lru_queue_display(); - - fn_exit: - return e->mr; - fn_fail: - goto fn_exit; -#endif -} - -static void ibcom_reg_mr_dereg(struct ibv_mr *mr) -{ - - struct ibcom_reg_mr_cache_entry_t *e; - struct ibcom_reg_mr_cache_entry_t *zero = 0; - unsigned long offset = (unsigned long) zero->mr; - e = (struct ibcom_reg_mr_cache_entry_t *) ((unsigned long) mr - offset); - e->refc--; - - //dprintf("ibcom_reg_mr_dereg,entry=%p,mr=%p,addr=%p,refc=%d,offset=%lx\n", e, mr, e->mr->addr, - //e->refc, offset); -} - -void ibcom_RegisterCacheInit() -{ - int i; - - /* Using the address to the start node to express the end of the list - * instead of using NULL */ - for (i = 0; i < IBCOM_REG_MR_NLINE; i++) { - ibcom_reg_mr_cache[i].lru_next = (struct ibcom_reg_mr_listnode_t *) &ibcom_reg_mr_cache[i]; - ibcom_reg_mr_cache[i].lru_prev = (struct ibcom_reg_mr_listnode_t *) &ibcom_reg_mr_cache[i]; - } - - dprintf("[MrCache] cache initializes %d entries\n", IBCOM_REG_MR_NLINE); -} - -void ibcom_RegisterCacheDestroy() -{ - struct ibcom_reg_mr_cache_entry_t *p; - int i = 0, cnt = 0; - - for (i = 0; i < IBCOM_REG_MR_NLINE; i++) { - for (p = (struct ibcom_reg_mr_cache_entry_t *) ibcom_reg_mr_cache[i].lru_next; - p != (struct ibcom_reg_mr_cache_entry_t *) &ibcom_reg_mr_cache[i]; - p = (struct ibcom_reg_mr_cache_entry_t *) p->lru_next) { - if (p && p->addr > 0) { - ibcom_dereg_mr(p->mr); - afree(p, IBCOM_AALLOC_ID_MRCACHE); - cnt++; - } - } - } - - //__lru_queue_display(); - - dprintf("[MrCache] cache destroyed %d entries\n", cnt); -} diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/errnames.txt b/src/mpid/ch3/channels/nemesis/netmod/dcfa/errnames.txt deleted file mode 100644 index c6b16ef..0000000 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/errnames.txt +++ /dev/null @@ -1,53 +0,0 @@ - -**MPIDI_PG_GetConnKVSname:MPIDI_PG_GetConnKVSname failed -**MPID_nem_dcfa_cm_accept:MPID_nem_dcfa_cm_accept failed -**MPID_nem_dcfa_cm_drain_scq:MPID_nem_dcfa_cm_drain_scq failed -**MPID_nem_dcfa_cm_poll:MPID_nem_dcfa_cm_poll failed -**MPID_nem_dcfa_cm_put:MPID_nem_dcfa_cm_put failed -**MPID_nem_dcfa_cm_send_core:MPID_nem_dcfa_cm_send_core failed -**MPID_nem_dcfa_drain_scq:MPID_nem_dcfa_drain_scq failed -**MPID_nem_dcfa_drain_scq_lmt_put:MPID_nem_dcfa_drain_scq_lmt_put failed -**MPID_nem_dcfa_drain_scq_scratch_pad:MPID_nem_dcfa_drain_scq_scratch_pad failed -**MPID_nem_dcfa_kvs_put_binary:MPID_nem_dcfa_kvs_put_binary failed -**MPID_nem_dcfa_lmt_done_recv:MPID_nem_dcfa_lmt_done_recv failed -**MPID_nem_dcfa_lmt_done_send:MPID_nem_dcfa_lmt_done_send failed -**MPID_nem_dcfa_npollingset:MPID_nem_dcfa_npollingset failed -**MPID_nem_dcfa_poll:MPID_nem_dcfa_poll failed -**MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state:MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state failed -**MPID_nem_dcfa_send_reply_seq_num:MPID_nem_dcfa_send_reply_seq_num failed -**MPID_nem_dcfa_send_req_seq_num:MPID_nem_dcfa_send_req_seq_num failed -**PMI_Barrier:PMI_Barrier failed -**PMI_KVS_Get:PMI_KVS_Get failed -**PMI_KVS_Put:PMI_KVS_Put failed -**PMS_KVS_Get:PMS_KVS_Get failed -**donepkt:donepkt failed -**ibcomOpen:ibcomOpen failed -**ibcom_alloc:ibcom_alloc failed -**ibcom_close:ibcom_close failed -**ibcom_get_info_conn:ibcom_get_info_conn failed -**ibcom_get_info_mr:ibcom_get_info_mr failed -**ibcom_irecv:ibcom_irecv failed -**ibcom_isend:ibcom_isend failed -**ibcom_lrecv:ibcom_lrecv failed -**ibcom_lsr_seq_num_tail_get:ibcom_lsr_seq_num_tail_get failed -**ibcom_mem_udwr_from:ibcom_mem_udwr_from failed -**ibcom_mem_udwr_to:ibcom_mem_udwr_to failed -**ibcom_obtain_pointer:ibcom_obtain_pointer failed -**ibcom_poll_cq %s:ibcom_poll_cq failed with error %s -**ibcom_poll_cq:ibcom_poll_cq failed -**ibcom_put_scratch_pad:ibcom_put_scratch_pad failed -**ibcom_rdmabuf_occupancy_notify_lstate_get:ibcom_rdmabuf_occupancy_notify_lstate_get failed -**ibcom_rdmabuf_occupancy_notify_rate_get:ibcom_rdmabuf_occupancy_notify_rate_get failed -**ibcom_rdmabuf_occupancy_notify_rstate_get:ibcom_rdmabuf_occupancy_notify_rstate_get failed -**ibcom_reg_mr_connect:ibcom_reg_mr_connect failed -**ibcom_reg_mr_fetch:ibcom_reg_mr_fetch failed -**ibcom_rsr_seq_num_tail_get:ibcom_rsr_seq_num_tail_get failed -**ibcom_rsr_seq_num_tail_last_sent_get:ibcom_rsr_seq_num_tail_last_sent_get failed -**ibcom_rts:ibcom_rts failed -**ibcom_sq_occupancy_notify_rate_get:ibcom_sq_occupancy_notify_rate_get failed -**ibcom_sseq_num_get:ibcom_sseq_num_get failed -**ibcom_udrecv:ibcom_udrecv failed -**netmod,dcfa,ibv_poll_cq:netmod,dcfa,ibv_poll_cq failed -**notimplemented:notimplemented failed -**outofmemory:outofmemory failed -**sizeof(MPIDI_CH3_Pkt_t):sizeof(MPIDI_CH3_Pkt_t) failed diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk b/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk new file mode 100644 index 0000000..22854b0 --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk @@ -0,0 +1,22 @@ +## -*- Mode: Makefile; -*- +## vim: set ft=automake : +## +## (C) 2011 by Argonne National Laboratory. +## See COPYRIGHT in top-level directory. +## + +if BUILD_NEMESIS_NETMOD_IB + +lib_lib@MPILIBNAME@_la_SOURCES += \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c + +noinst_HEADERS += \ + src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h + +endif BUILD_NEMESIS_NETMOD_IB diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/cross_values.txt b/src/mpid/ch3/channels/nemesis/netmod/ib/cross_values.txt similarity index 100% rename from src/mpid/ch3/channels/nemesis/netmod/dcfa/cross_values.txt rename to src/mpid/ch3/channels/nemesis/netmod/ib/cross_values.txt diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/errnames.txt b/src/mpid/ch3/channels/nemesis/netmod/ib/errnames.txt new file mode 100644 index 0000000..e71e98e --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/errnames.txt @@ -0,0 +1,53 @@ + +**MPIDI_PG_GetConnKVSname:MPIDI_PG_GetConnKVSname failed +**MPID_nem_ib_cm_accept:MPID_nem_ib_cm_accept failed +**MPID_nem_ib_cm_drain_scq:MPID_nem_ib_cm_drain_scq failed +**MPID_nem_ib_cm_poll:MPID_nem_ib_cm_poll failed +**MPID_nem_ib_cm_put:MPID_nem_ib_cm_put failed +**MPID_nem_ib_cm_send_core:MPID_nem_ib_cm_send_core failed +**MPID_nem_ib_drain_scq:MPID_nem_ib_drain_scq failed +**MPID_nem_ib_drain_scq_lmt_put:MPID_nem_ib_drain_scq_lmt_put failed +**MPID_nem_ib_drain_scq_scratch_pad:MPID_nem_ib_drain_scq_scratch_pad failed +**MPID_nem_ib_kvs_put_binary:MPID_nem_ib_kvs_put_binary failed +**MPID_nem_ib_lmt_done_recv:MPID_nem_ib_lmt_done_recv failed +**MPID_nem_ib_lmt_done_send:MPID_nem_ib_lmt_done_send failed +**MPID_nem_ib_npollingset:MPID_nem_ib_npollingset failed +**MPID_nem_ib_poll:MPID_nem_ib_poll failed +**MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state:MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state failed +**MPID_nem_ib_send_reply_seq_num:MPID_nem_ib_send_reply_seq_num failed +**MPID_nem_ib_send_req_seq_num:MPID_nem_ib_send_req_seq_num failed +**PMI_Barrier:PMI_Barrier failed +**PMI_KVS_Get:PMI_KVS_Get failed +**PMI_KVS_Put:PMI_KVS_Put failed +**PMS_KVS_Get:PMS_KVS_Get failed +**MPID_nem_ib_lmt_send_GET_DONE:MPID_nem_ib_lmt_send_GET_DONE failed +**MPID_nem_ib_com_open:MPID_nem_ib_com_open failed +**MPID_nem_ib_com_alloc:MPID_nem_ib_com_alloc failed +**MPID_nem_ib_com_close:MPID_nem_ib_com_close failed +**MPID_nem_ib_com_get_info_conn:MPID_nem_ib_com_get_info_conn failed +**MPID_nem_ib_com_get_info_mr:MPID_nem_ib_com_get_info_mr failed +**MPID_nem_ib_com_irecv:MPID_nem_ib_com_irecv failed +**MPID_nem_ib_com_isend:MPID_nem_ib_com_isend failed +**MPID_nem_ib_com_lrecv:MPID_nem_ib_com_lrecv failed +**MPID_nem_ib_com_lsr_seq_num_tail_get:MPID_nem_ib_com_lsr_seq_num_tail_get failed +**MPID_nem_ib_com_mem_udwr_from:MPID_nem_ib_com_mem_udwr_from failed +**MPID_nem_ib_com_mem_udwr_to:MPID_nem_ib_com_mem_udwr_to failed +**MPID_nem_ib_com_obtain_pointer:MPID_nem_ib_com_obtain_pointer failed +**MPID_nem_ib_com_poll_cq %s:MPID_nem_ib_com_poll_cq failed with error %s +**MPID_nem_ib_com_poll_cq:MPID_nem_ib_com_poll_cq failed +**MPID_nem_ib_com_put_scratch_pad:MPID_nem_ib_com_put_scratch_pad failed +**MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get:MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get failed +**MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get:MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get failed +**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get:MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get failed +**MPID_nem_ib_com_reg_mr_connect:MPID_nem_ib_com_reg_mr_connect failed +**MPID_nem_ib_com_reg_mr_fetch:MPID_nem_ib_com_reg_mr_fetch failed +**MPID_nem_ib_com_rsr_seq_num_tail_get:MPID_nem_ib_com_rsr_seq_num_tail_get failed +**MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get:MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get failed +**MPID_nem_ib_com_rts:MPID_nem_ib_com_rts failed +**MPID_nem_ib_com_sq_occupancy_notify_rate_get:MPID_nem_ib_com_sq_occupancy_notify_rate_get failed +**MPID_nem_ib_com_sseq_num_get:MPID_nem_ib_com_sseq_num_get failed +**MPID_nem_ib_com_udrecv:MPID_nem_ib_com_udrecv failed +**netmod,ib,ibv_poll_cq:netmod,ib,ibv_poll_cq failed +**notimplemented:notimplemented failed +**outofmemory:outofmemory failed +**sizeof(MPIDI_CH3_Pkt_t):sizeof(MPIDI_CH3_Pkt_t) failed diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c similarity index 57% rename from src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c rename to src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c index 5b10045..3dd2881 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c @@ -5,40 +5,40 @@ * See COPYRIGHT in top-level directory. */ -#include "dcfa_impl.h" +#include "ib_impl.h" -//#define DEBUG_DCFA_FINALIZE +//#define MPID_NEM_IB_DEBUG_FINALIZE #ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ #undef dprintf #endif -#ifdef DEBUG_DCFA_FINALIZE +#ifdef MPID_NEM_IB_DEBUG_FINALIZE #define dprintf printf #else #define dprintf(...) #endif #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_finalize +#define FUNCNAME MPID_nem_ib_finalize #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_finalize(void) +int MPID_nem_ib_finalize(void) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; int i; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_FINALIZE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_FINALIZE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_FINALIZE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_FINALIZE); #if 0 - for (i = 0; i < MPID_nem_dcfa_nranks; i++) { - ibcom_errno = ibcom_close(MPID_nem_dcfa_conns[i].fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_close"); + for (i = 0; i < MPID_nem_ib_nranks; i++) { + ibcom_errno = MPID_nem_ib_com_close(MPID_nem_ib_conns[i].fd); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_close"); } #endif - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_FINALIZE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_FINALIZE); fn_exit: return mpi_errno; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c new file mode 100644 index 0000000..ce2db17 --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c @@ -0,0 +1,2217 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2012 NEC Corporation + * Author: Masamichi Takagi + * (C) 2012 Oct 14 Yutaka Ishikawa, ishikawa(a)is.s.u-tokyo.ac.jp + * See COPYRIGHT in top-level directory. + */ + +/* + * TODO: + * - MPID_nem_ib_com_clean might not clean all allocated memory area. Need to FIX it. + * - During error processing in each function, some memory area might not + * be deallocated. Look at all functions. + */ +#include "ib_ibcom.h" +#include <sys/ipc.h> +#include <sys/shm.h> +#include <sys/types.h> +#include <assert.h> +#include <linux/mman.h> /* make it define MAP_ANONYMOUS */ +#include <sys/mman.h> + +//#define MPID_NEM_IB_DEBUG_IBCOM +#ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ +#undef dprintf +#endif +#ifdef MPID_NEM_IB_DEBUG_IBCOM +#define dprintf printf +#else +#define dprintf(...) +#endif + +int dflag; + +static int sendwr_id = 10; +static MPID_nem_ib_com_t contab[MPID_NEM_IB_COM_SIZE]; +static int ib_initialized = 0; +static int maxcon; +static struct ibv_device **ib_devlist; +static struct ibv_context *ib_ctx; +struct ibv_context *MPID_nem_ib_ctx_export; /* for SC13 demo connector */ +static struct ibv_pd *ib_pd; +struct ibv_pd *MPID_nem_ib_pd_export; /* for SC13 demo connector */ +struct ibv_cq *MPID_nem_ib_rc_shared_scq; +struct ibv_cq *MPID_nem_ib_rc_shared_scq_lmt_put; +struct ibv_cq *MPID_nem_ib_rc_shared_scq_scratch_pad; +static struct ibv_cq *MPID_nem_ib_rc_shared_rcq; +static struct ibv_cq *MPID_nem_ib_rc_shared_rcq_lmt_put; +static struct ibv_cq *MPID_nem_ib_rc_shared_rcq_scratch_pad; +static struct ibv_cq *MPID_nem_ib_ud_shared_scq; +struct ibv_cq *MPID_nem_ib_ud_shared_rcq; +static uint8_t *scratch_pad = 0; + +#define MPID_NEM_IB_RANGE_CHECK(condesc, conp) \ +{ \ + if (condesc < 0 || condesc >= MPID_NEM_IB_COM_SIZE) return; \ + conp = &contab[condesc]; \ + if (conp->icom_used != 1) return; \ +} + +#define MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp) \ +{ \ + if (condesc < 0 || condesc >= MPID_NEM_IB_COM_SIZE) { \ + return -1; \ + } \ + conp = &contab[condesc]; \ + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_used != 1, -1, dprintf("MPID_NEM_IB_RANGE_CHECK_WITH_ERROR,conp->icom_used=%d\n", conp->icom_used)); \ +} + +static int modify_qp_to_init(struct ibv_qp *qp, int ib_port) +{ + struct ibv_qp_attr attr; + int flags; + int rc; + + memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_INIT; + attr.port_num = ib_port; + attr.pkey_index = 0; + attr.qp_access_flags = + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS; + rc = ibv_modify_qp(qp, &attr, flags); + if (rc) { + fprintf(stderr, "failed to modify QP state to INIT\n"); + } + return rc; +} + +static int modify_qp_to_rtr(struct ibv_qp *qp, uint32_t remote_qpn, uint16_t dlid, + union ibv_gid *dgid, int ib_port, int gid_idx) +{ + struct ibv_qp_attr attr; + int flags; + int rc; + + memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_RTR; + attr.path_mtu = IBV_MTU_2048; + //attr.path_mtu = IBV_MTU_1024; + //attr.path_mtu = IBV_MTU_256; /* DCFA */ + attr.dest_qp_num = remote_qpn; + attr.rq_psn = 0; + attr.max_dest_rd_atomic = MPID_NEM_IB_COM_MAX_RD_ATOMIC; + //attr.max_dest_rd_atomic = 1; + //attr.max_dest_rd_atomic = 0; /* DCFA */ + + /* Default is 0x12 (= 5.12ms) see IB Spec. Rel. 1.2, Vol. 1, 9.7.5.2.8 */ + attr.min_rnr_timer = 0x12; + + attr.ah_attr.dlid = dlid; + attr.ah_attr.sl = 0; + attr.ah_attr.src_path_bits = 0; + attr.ah_attr.is_global = 0; + attr.ah_attr.port_num = ib_port; + + /* In dcfa gid is not set and for testing here it is also not set */ +#if 1 +#ifdef HAVE_LIBDCFA /* DCFA doesn't use gid */ +#else + if (gid_idx >= 0) { + attr.ah_attr.is_global = 1; + attr.ah_attr.port_num = ib_port; + memcpy(&attr.ah_attr.grh.dgid, dgid, 16); + attr.ah_attr.grh.flow_label = 0; + attr.ah_attr.grh.hop_limit = 1; + attr.ah_attr.grh.sgid_index = gid_idx; + attr.ah_attr.grh.traffic_class = 0; + } +#endif +#endif + + flags = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN + | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; + rc = ibv_modify_qp(qp, &attr, flags); + if (rc) { + dprintf("failed to modify QP state to RTR\n"); + } + return rc; +} + +static int modify_qp_to_rts(struct ibv_qp *qp) +{ + struct ibv_qp_attr attr; + int flags; + int rc; + + memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_RTS; + attr.timeout = (0x14); /* timeout 4.096us * 2^x */ + attr.retry_cnt = 7; + attr.rnr_retry = 7; + attr.sq_psn = 0; + attr.max_rd_atomic = MPID_NEM_IB_COM_MAX_RD_ATOMIC; + //attr.max_rd_atomic = 1; + //attr.max_rd_atomic = 0; /* DCFA */ + + flags = IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT + | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC; + rc = ibv_modify_qp(qp, &attr, flags); + if (rc) { + fprintf(stderr, "failed to modify QP state to RTS\n"); + } + return rc; +} + +/* called from MPID_nem_ib_com_open if needed */ +static int MPID_nem_ib_com_device_init() +{ + int ibcom_errno = 0; + int dev_num; + char *dev_name; + int i; + + if (ib_initialized == 1) { + dprintf("MPID_nem_ib_com_device_init,already initialized\n"); + return 0; + } + if (ib_initialized == -1) + return -1; + + /* initialize ibv_reg_mr cache */ + MPID_nem_ib_com_register_cache_init(); + + /* Get the device list */ + ib_devlist = ibv_get_device_list(&dev_num); + if (!ib_devlist || !dev_num) { + fprintf(stderr, "No IB device is found\n"); + return -1; + } + +#ifdef HAVE_LIBDCFA + for (i = 0; i < dev_num; i++) { + if (ib_devlist[i]) { + goto dev_found; + } + } +#else + for (i = 0; i < dev_num; i++) { + if (!strcmp(ibv_get_device_name(ib_devlist[i]), "mlx4_0")) { + goto dev_found; + } + } +#endif + MPID_NEM_IB_COM_ERR_SETANDJUMP(-1, printf("IB device not found")); + dev_found: + + /* Open the requested device */ + if (MPID_nem_ib_ctx_export) { + ib_ctx = MPID_nem_ib_ctx_export; + } + else { + ib_ctx = ibv_open_device(ib_devlist[i]); + } + dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_ctx_export=%p,ib_ctx=%p\n", MPID_nem_ib_ctx_export, ib_ctx); + if (!ib_ctx) { + fprintf(stderr, "failed to open IB device\n"); + goto err_exit; + } + MPID_nem_ib_ctx_export = ib_ctx; +#ifdef HAVE_LIBDCFA +#else + dev_name = strdup(ibv_get_device_name(ib_devlist[i])); + dprintf("MPID_nem_ib_com_device_init,dev_name=%s\n", dev_name); +#endif + /* Create a PD */ + if (MPID_nem_ib_pd_export) { + ib_pd = MPID_nem_ib_pd_export; + } + else { + ib_pd = ibv_alloc_pd(ib_ctx); + } + dprintf("MPID_nem_ib_com_device_init,MPID_nem_ib_pd_export=%p,ib_pd=%p\n", MPID_nem_ib_pd_export, ib_pd); + if (!ib_pd) { + fprintf(stderr, "ibv_alloc_pd failed\n"); + goto err_exit; + } + MPID_nem_ib_pd_export = ib_pd; + + ib_initialized = 1; + fn_exit: + return ibcom_errno; + + err_exit: + ib_initialized = -1; + if (ib_devlist) + ibv_free_device_list(ib_devlist); + if (ib_ctx) + ibv_close_device(ib_ctx); + return -1; + fn_fail: + goto fn_exit; +} + +static void MPID_nem_ib_com_clean(MPID_nem_ib_com_t * conp) +{ + int i; + + if (conp->icom_qp) + ibv_destroy_qp(conp->icom_qp); + if (conp->icom_mrlist && conp->icom_mrlen > 0) { + switch (conp->open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + for (i = 0; i < MPID_NEM_IB_COM_NBUF_RDMA; i++) { + if (conp->icom_mrlist[i]) { + ibv_dereg_mr(conp->icom_mrlist[i]); + } + } + break; + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + for (i = 0; i < MPID_NEM_IB_COM_NBUF_SCRATCH_PAD; i++) { + if (conp->icom_mrlist[i]) { + ibv_dereg_mr(conp->icom_mrlist[i]); + } + } + break; + case MPID_NEM_IB_COM_OPEN_UD: + for (i = 0; i < MPID_NEM_IB_COM_NBUF_UD; i++) { + if (conp->icom_mrlist[i]) { + ibv_dereg_mr(conp->icom_mrlist[i]); + } + } + break; + } + free(conp->icom_mrlist); + } + if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]) { + munmap(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ); + } + if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) { + munmap(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], MPID_NEM_IB_COM_RDMABUF_SZ); + } + if (conp->icom_scq) { + ibv_destroy_cq(conp->icom_scq); + } + if (conp->icom_rcq) { + ibv_destroy_cq(conp->icom_rcq); + } + if (conp->icom_rmem) { + free(conp->icom_rmem); + } + if (conp->icom_rsize) { + free(conp->icom_rsize); + } + if (conp->icom_rkey) { + free(conp->icom_rkey); + } + memset(conp, 0, sizeof(MPID_nem_ib_com_t)); + // TODO: free ah, sge, command template, ... +} + +int MPID_nem_ib_com_open(int ib_port, int open_flag, int *condesc) +{ + int ibcom_errno = 0, ib_errno; + MPID_nem_ib_com_t *conp; + struct ibv_qp_init_attr qp_init_attr; + struct ibv_sge *sge; + struct ibv_send_wr *sr; + struct ibv_recv_wr *rr, *bad_wr; + int mr_flags; + int i; + + dprintf("MPID_nem_ib_com_open,port=%d,flag=%08x\n", ib_port, open_flag); + + int open_flag_conn = open_flag; + if (open_flag_conn != MPID_NEM_IB_COM_OPEN_RC && + open_flag_conn != MPID_NEM_IB_COM_OPEN_RC_LMT_PUT && + open_flag_conn != MPID_NEM_IB_COM_OPEN_UD && open_flag_conn != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD) { + dprintf("MPID_nem_ib_com_open,bad flag\n"); + ibcom_errno = -1; + goto fn_fail; + } + + /* device open error */ + if (MPID_nem_ib_com_device_init() < 0) { + ibcom_errno = -1; + goto fn_fail; + } + + /* no more connection can be estabilished */ + if (maxcon == MPID_NEM_IB_COM_SIZE) { + ibcom_errno = -1; + goto fn_fail; + } + + for (*condesc = 0; *condesc < MPID_NEM_IB_COM_SIZE; (*condesc)++) { + //dprintf("*condesc=%d,used=%d\n", *condesc, contab[*condesc].icom_used); + if (contab[*condesc].icom_used == 0) { + goto ok_cont; + } + } + /* count says not full, but we couldn't fine vacant slot */ + dprintf("contable has inconsistent\n"); + ibcom_errno = -1; + goto fn_fail; + + ok_cont: + dprintf("MPID_nem_ib_com_open,condesc=%d\n", *condesc); + conp = &contab[*condesc]; + memset(conp, 0, sizeof(MPID_nem_ib_com_t)); + conp->icom_used = 1; + conp->icom_port = ib_port; + conp->open_flag = open_flag; + conp->rsr_seq_num_poll = 0; /* it means slot 0 is polled */ + conp->rsr_seq_num_tail = -1; /* it means slot 0 is not released */ + conp->rsr_seq_num_tail_last_sent = -1; + conp->lsr_seq_num_tail = -1; + conp->lsr_seq_num_tail_last_requested = -2; + conp->rdmabuf_occupancy_notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW; + conp->rdmabuf_occupancy_notify_lstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW; + //dprintf("MPID_nem_ib_com_open,ptr=%p,rsr_seq_num_poll=%d\n", conp, conp->rsr_seq_num_poll); + +#ifdef HAVE_LIBDCFA +#else + if (ibv_query_port(ib_ctx, ib_port, &conp->icom_pattr)) { + dprintf("ibv_query_port on port %u failed\n", ib_port); + goto err_exit; + } +#endif + + /* Create send/recv CQ */ + switch (open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + if (!MPID_nem_ib_rc_shared_scq) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_rc_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_rc_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq, -1, dprintf("MPID_nem_ib_rc_shared_scq")); + } + conp->icom_scq = MPID_nem_ib_rc_shared_scq; + + if (!MPID_nem_ib_rc_shared_rcq) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_rc_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_rc_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq, -1, dprintf("MPID_nem_ib_rc_shared_rcq")); + } + conp->icom_rcq = MPID_nem_ib_rc_shared_rcq; + break; + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + if (!MPID_nem_ib_rc_shared_scq_scratch_pad) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_rc_shared_scq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_scratch_pad, -1, dprintf("MPID_nem_ib_rc_shared_scq")); + } + conp->icom_scq = MPID_nem_ib_rc_shared_scq_scratch_pad; + + if (!MPID_nem_ib_rc_shared_rcq_scratch_pad) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_rc_shared_rcq_scratch_pad = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_scratch_pad, -1, dprintf("MPID_nem_ib_rc_shared_rcq")); + } + conp->icom_rcq = MPID_nem_ib_rc_shared_rcq_scratch_pad; + break; + case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: + if (!MPID_nem_ib_rc_shared_scq_lmt_put) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_rc_shared_scq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_scq_lmt_put, -1, dprintf("MPID_nem_ib_rc_shared_scq")); + } + conp->icom_scq = MPID_nem_ib_rc_shared_scq_lmt_put; + + if (!MPID_nem_ib_rc_shared_rcq_lmt_put) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_rc_shared_rcq_lmt_put = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_rc_shared_rcq_lmt_put, -1, dprintf("MPID_nem_ib_rc_shared_rcq")); + } + conp->icom_rcq = MPID_nem_ib_rc_shared_rcq_lmt_put; + break; + case MPID_NEM_IB_COM_OPEN_UD: + if (!MPID_nem_ib_ud_shared_scq) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_ud_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_ud_shared_scq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_scq, -1, dprintf("MPID_nem_ib_ud_shared_scq")); + } + conp->icom_scq = MPID_nem_ib_ud_shared_scq; + + if (!MPID_nem_ib_ud_shared_rcq) { +#ifdef HAVE_LIBDCFA + MPID_nem_ib_ud_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY); +#else + MPID_nem_ib_ud_shared_rcq = ibv_create_cq(ib_ctx, MPID_NEM_IB_COM_MAX_CQ_CAPACITY, NULL, NULL, 0); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!MPID_nem_ib_ud_shared_rcq, -1, dprintf("MPID_nem_ib_ud_shared_rcq")); + } + conp->icom_rcq = MPID_nem_ib_ud_shared_rcq; + break; + } + + /* Create QP */ + memset(&qp_init_attr, 0, sizeof(qp_init_attr)); + qp_init_attr.send_cq = conp->icom_scq; + qp_init_attr.recv_cq = conp->icom_rcq; + qp_init_attr.cap.max_send_wr = MPID_NEM_IB_COM_MAX_SQ_CAPACITY; + qp_init_attr.cap.max_recv_wr = MPID_NEM_IB_COM_MAX_RQ_CAPACITY; + qp_init_attr.cap.max_send_sge = MPID_NEM_IB_COM_MAX_SGE_CAPACITY; + qp_init_attr.cap.max_recv_sge = MPID_NEM_IB_COM_MAX_SGE_CAPACITY; + qp_init_attr.cap.max_inline_data = MPID_NEM_IB_COM_INLINE_DATA; + switch (open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + qp_init_attr.qp_type = IBV_QPT_RC; + break; + case MPID_NEM_IB_COM_OPEN_UD: + qp_init_attr.qp_type = IBV_QPT_UD; + break; + default: + MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, dprintf("invalid open_flag\n")); + break; + } + qp_init_attr.sq_sig_all = 1; + + conp->icom_qp = ibv_create_qp(ib_pd, &qp_init_attr); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_qp, -1, printf("ibv_create_qp\n")); + + conp->max_send_wr = qp_init_attr.cap.max_send_wr; + conp->max_recv_wr = qp_init_attr.cap.max_recv_wr; + conp->max_inline_data = qp_init_attr.cap.max_inline_data; + + dprintf("MPID_nem_ib_com_open,max_send_wr=%d,max_recv_wr=%d,max_inline_data=%d\n", + qp_init_attr.cap.max_send_wr, qp_init_attr.cap.max_recv_wr, + qp_init_attr.cap.max_inline_data); + dprintf("MPID_nem_ib_com_open,fd=%d,qpn=%08x\n", *condesc, conp->icom_qp->qp_num); +#ifdef HAVE_LIBDCFA + dprintf("MPID_nem_ib_com_open,fd=%d,lid=%04x\n", *condesc, ib_ctx->lid); +#else + dprintf("MPID_nem_ib_com_open,fd=%d,lid=%04x\n", *condesc, conp->icom_pattr.lid); +#endif + +#ifdef HAVE_LIBDCFA + /* DCFA doesn't use gid */ + for (i = 0; i < 16; i++) { + conp->icom_gid.raw[i] = 0; + } +#else + ib_errno = ibv_query_gid(ib_ctx, ib_port, 0, &conp->icom_gid); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibv_query_gid\n")); + + dprintf("MPID_nem_ib_com_open,fd=%d,my_gid=", *condesc); + for (i = 0; i < 16; i++) { + dprintf("%02x", (int) conp->icom_gid.raw[i]); + } + dprintf("\n"); +#endif + + /* buffers */ + switch (open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + /* RDMA-write-from and -to local memory area */ + conp->icom_mrlist = malloc(sizeof(struct ibv_mr *) * MPID_NEM_IB_COM_NBUF_RDMA); + memset(conp->icom_mrlist, 0, sizeof(struct ibv_mr *) * MPID_NEM_IB_COM_NBUF_RDMA); + conp->icom_mrlen = MPID_NEM_IB_COM_NBUF_RDMA; + conp->icom_mem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); + //printf("open,icom_mem=%p\n", conp->icom_mem); + memset(conp->icom_mem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); + conp->icom_msize = (int *) malloc(sizeof(int *) * MPID_NEM_IB_COM_NBUF_RDMA); + memset(conp->icom_msize, 0, sizeof(int *) * MPID_NEM_IB_COM_NBUF_RDMA); + mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + + /* RDMA-write-from local memory area */ + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM] = MPID_NEM_IB_COM_RDMABUF_SZ; + conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] = + mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], MPID_NEM_IB_COM_RDMABUF_SZ); + if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] == (void *) -1) { + fprintf(stderr, "failed to allocate buffer\n"); + goto err_exit; + } + memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], 0, conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]); + + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM] = + MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM], + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_FROM]); + if (!conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]) { + fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); + goto err_exit; + } + + /* RDMA-write-to local memory area */ + conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO] = MPID_NEM_IB_COM_RDMABUF_SZ; +#if 0 + int shmid = shmget(2, MPID_NEM_IB_COM_RDMABUF_SZ, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(shmid < 0, -1, perror("shmget")); + conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] = shmat(shmid, 0, 0); + if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] == (char *) -1) { + perror("Shared memory attach failure"); + shmctl(shmid, IPC_RMID, NULL); + ibcom_errno = -1; + goto fn_fail; + } +#else + conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] = + mmap(0, MPID_NEM_IB_COM_RDMABUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], MPID_NEM_IB_COM_RDMABUF_SZ); +#endif + if (conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] == (void *) -1) { + fprintf(stderr, "failed to allocate buffer\n"); + goto err_exit; + } + memset(conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], 0, conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO]); + + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO] = + ibv_reg_mr(ib_pd, conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], conp->icom_msize[MPID_NEM_IB_COM_RDMAWR_TO], + mr_flags); + if (!conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]) { + fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); + goto err_exit; + } +#ifdef HAVE_LIBDCFA + dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->buf); +#else + dprintf("MPID_nem_ib_com_open,fd=%d,rmem=%p\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->addr); +#endif + dprintf("MPID_nem_ib_com_open,fd=%d,rkey=%08x\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_TO]->rkey); + + /* RDMA-write-to remote memory area */ + conp->icom_rmem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); + if (conp->icom_rmem == 0) + goto err_exit; + memset(conp->icom_rmem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); + + conp->icom_rsize = (size_t *) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); + if (conp->icom_rsize == 0) + goto err_exit; + memset(conp->icom_rsize, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_RDMA); + + conp->icom_rkey = (int *) malloc(sizeof(int) * MPID_NEM_IB_COM_NBUF_RDMA); + if (conp->icom_rkey == 0) + goto err_exit; + memset(conp->icom_rkey, 0, sizeof(int) * MPID_NEM_IB_COM_NBUF_RDMA); + break; + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + /* RDMA-write-from and -to local memory area */ + conp->icom_mrlist = malloc(sizeof(struct ibv_mr *) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + memset(conp->icom_mrlist, 0, sizeof(struct ibv_mr *) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + conp->icom_mrlen = MPID_NEM_IB_COM_NBUF_SCRATCH_PAD; + conp->icom_mem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + memset(conp->icom_mem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + conp->icom_msize = (int *) malloc(sizeof(int *) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + memset(conp->icom_msize, 0, sizeof(int *) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + + /* RDMA-write-to remote memory area */ + conp->icom_rmem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_rmem == 0, -1, dprintf("malloc failed\n")); + memset(conp->icom_rmem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + + conp->icom_rsize = (size_t *) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_rsize == 0, -1, dprintf("malloc failed\n")); + memset(conp->icom_rsize, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + + conp->icom_rkey = (int *) malloc(sizeof(int) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_rkey == 0, -1, dprintf("malloc failed\n")); + memset(conp->icom_rkey, 0, sizeof(int) * MPID_NEM_IB_COM_NBUF_SCRATCH_PAD); + break; + + case MPID_NEM_IB_COM_OPEN_UD: + /* UD-write-from and -to local memory area */ + conp->icom_mrlist = malloc(sizeof(struct ibv_mr *) * MPID_NEM_IB_COM_NBUF_UD); + memset(conp->icom_mrlist, 0, sizeof(struct ibv_mr *) * MPID_NEM_IB_COM_NBUF_UD); + conp->icom_mrlen = MPID_NEM_IB_COM_NBUF_UD; + conp->icom_mem = (void **) malloc(sizeof(void **) * MPID_NEM_IB_COM_NBUF_UD); + memset(conp->icom_mem, 0, sizeof(void **) * MPID_NEM_IB_COM_NBUF_UD); + conp->icom_msize = (int *) malloc(sizeof(int *) * MPID_NEM_IB_COM_NBUF_UD); + memset(conp->icom_msize, 0, sizeof(int *) * MPID_NEM_IB_COM_NBUF_UD); + mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + + /* UD-write-from local memory area */ + MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n")); + conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM] = MPID_NEM_IB_COM_UDBUF_SZ; + conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] = + mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], MPID_NEM_IB_COM_UDBUF_SZ); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] == (void *) -1, -1, + dprintf("failed to allocate buffer\n")); + memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], 0, conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]); + + conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM] = + MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM], conp->icom_msize[MPID_NEM_IB_COM_UDWR_FROM]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM], -1, + dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + + /* UD-write-to local memory area */ + /* addr to addr+39 are not filled, addr+40 to addr+length-1 are filled with payload */ + MPID_NEM_IB_COM_ERR_CHKANDJUMP(MPID_NEM_IB_COM_UDBUF_SZ <= 40, -1, dprintf("buf_size too short\n")); + conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO] = MPID_NEM_IB_COM_UDBUF_SZ; + conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO] = + mmap(0, MPID_NEM_IB_COM_UDBUF_SZ, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + dprintf("MPID_nem_ib_com_open,mmap=%p,len=%d\n", conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], MPID_NEM_IB_COM_UDBUF_SZ); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO] == (void *) -1, -1, + dprintf("failed to allocate buffer\n")); + memset(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], 0, conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]); + + conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO] = + MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO], conp->icom_msize[MPID_NEM_IB_COM_UDWR_TO]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO], -1, + dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + + /* initialize arena allocator for MPID_NEM_IB_COM_UDWR_TO */ + //MPID_nem_ib_com_udbuf_init(conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO]); + + dprintf("MPID_nem_ib_com_open,ud,fd=%d,lkey=%08x\n", *condesc, conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO]->lkey); + break; + default: + MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, dprintf("invalid open_flag\n")); + break; + + } + + /* command templates */ + switch (open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + + /* SR (send request) template */ + conp->icom_sr = + (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_NTEMPLATE); + memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_NTEMPLATE); + + for (i = 0; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { + /* SGE (RDMA-send-from memory) template */ +#ifdef HAVE_LIBDCFA + memset(&(conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[0]), 0, + sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = + (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE); + memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE); +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].next = + (i == + MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) ? NULL : &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + + i + 1]; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].opcode = IBV_WR_RDMA_WRITE; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].send_flags = + IBV_SEND_SIGNALED | IBV_SEND_INLINE; + } + + { +#ifdef HAVE_LIBDCFA + memset(&(conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[0]), 0, + sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = + (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * + MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE); + memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE); +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].opcode = IBV_WR_RDMA_WRITE; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags = IBV_SEND_SIGNALED; + } + { + /* SR (send request) template for MPID_NEM_IB_COM_LMT_INITIATOR */ +#ifdef HAVE_LIBDCFA + memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list[0]), 0, + sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = + (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_INITIATOR_NSGE); + memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_INITIATOR_NSGE); +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].opcode = IBV_WR_RDMA_READ; + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].send_flags = IBV_SEND_SIGNALED; + } + + /* SR (send request) template for MPID_NEM_IB_COM_LMT_PUT *//* for lmt-put-done */ +#ifdef HAVE_LIBDCFA + memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); + memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].opcode = IBV_WR_RDMA_WRITE; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].send_flags = IBV_SEND_SIGNALED; + + /* SR (send request) template for MPID_NEM_IB_COM_RDMAWR_FRMFIXED */ + /* not implemented */ + + /* SGE (scatter gather element) template for recv */ + /* nothing is required for RDMA-write */ + + /* RR (receive request) template for MPID_NEM_IB_COM_RDMAWR_RESPONDER */ + conp->icom_rr = + (struct ibv_recv_wr *) malloc(sizeof(struct ibv_recv_wr) * MPID_NEM_IB_COM_RC_RR_NTEMPLATE); + memset(conp->icom_rr, 0, sizeof(struct ibv_recv_wr) * MPID_NEM_IB_COM_RC_RR_NTEMPLATE); + + /* create one dummy RR to ibv_post_recv */ + conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER].sg_list = NULL; +#endif + conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER].num_sge = 0; + break; + + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD:{ + /* SR (send request) template */ + conp->icom_sr = + (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * + MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE); + memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE); + + /* SR (send request) template for MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR */ +#ifdef HAVE_LIBDCFA + memset(&(conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0]), 0, + sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = + (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * + MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE); + memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE); +#endif + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].num_sge = 1; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].opcode = IBV_WR_RDMA_WRITE; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].send_flags = IBV_SEND_SIGNALED; + break; + } + + case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: + /* SR (send request) template */ + conp->icom_sr = + (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * + MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE); + memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE); + /* SR (send request) template for MPID_NEM_IB_COM_LMT_PUT */ +#ifdef HAVE_LIBDCFA + memset(&(conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); + memset(sge, 0, sizeof(struct ibv_sge) * MPID_NEM_IB_COM_LMT_PUT_NSGE); +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].opcode = IBV_WR_RDMA_WRITE; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].send_flags = IBV_SEND_SIGNALED; + break; + + case MPID_NEM_IB_COM_OPEN_UD: + /* SGE (RDMA-send-from memory) template for MPID_NEM_IB_COM_UD_INITIATOR */ +#ifdef HAVE_LIBDCFA + sge = &(conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].sg_list[0]); + memset(sge, 0, sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = (struct ibv_sge *) calloc(1, sizeof(struct ibv_sge)); +#endif + /* addr to addr + length - 1 will be on the payload, but search backword for "<= 40" */ + sge[0].addr = (uint64_t) conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM] + 40; + sge[0].length = MPID_NEM_IB_COM_UDBUF_SZSEG - 40; + sge[0].lkey = conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_FROM]->lkey; + + + conp->icom_ah_attr = + (struct ibv_ah_attr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, sizeof(struct ibv_ah_attr)); + + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].sl = 0; + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].src_path_bits = 0; + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].static_rate = 0; /* not limit on static rate (100% port speed) */ + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 0; + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].port_num = conp->icom_port; + +#if 0 + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].is_global = 1; + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.flow_label = 0; + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.sgid_index = 0; /* what is this? */ + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.hop_limit = 1; + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.traffic_class = 0; +#endif + + /* SR (send request) template for MPID_NEM_IB_COM_UD_INITIATOR */ + conp->icom_sr = + (struct ibv_send_wr *) calloc(MPID_NEM_IB_COM_UD_SR_NTEMPLATE, sizeof(struct ibv_send_wr)); + + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].sg_list = sge; +#endif + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].num_sge = 1; + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].opcode = IBV_WR_SEND; + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].send_flags = IBV_SEND_SIGNALED; + + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].wr.ud.remote_qkey = MPID_NEM_IB_COM_QKEY; + + /* SGE (scatter gather element) template for recv */ +#ifdef HAVE_LIBDCFA + sge = &(conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].sg_list[0]); + memset(sge, 0, sizeof(struct ibv_sge) * WR_SG_NUM); +#else + sge = (struct ibv_sge *) calloc(1, sizeof(struct ibv_sge)); +#endif + sge[0].addr = (uint64_t) conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO]; + sge[0].length = MPID_NEM_IB_COM_UDBUF_SZ; + sge[0].lkey = conp->icom_mrlist[MPID_NEM_IB_COM_UDWR_TO]->lkey; + + /* RR (receive request) template for MPID_NEM_IB_COM_UD_RESPONDER */ + conp->icom_rr = + (struct ibv_recv_wr *) calloc(MPID_NEM_IB_COM_UD_RR_NTEMPLATE, sizeof(struct ibv_recv_wr)); + + /* create one dummy RR to ibv_post_recv */ + conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].next = NULL; +#ifdef HAVE_LIBDCFA +#else + conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].sg_list = sge; +#endif + conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].num_sge = 1; + break; + } + + maxcon++; + + fn_exit: + return ibcom_errno; + err_exit: + MPID_nem_ib_com_clean(conp); + return -1; + fn_fail: + MPID_nem_ib_com_clean(conp); + goto fn_exit; +} + +/* 1. allocate memory area if it's not allocated or reuse it if it's allocated + 2. ibv_reg_mr it and store rkey to conp->icom_mrlist + buf is output */ +int MPID_nem_ib_com_alloc(int condesc, int sz) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + int mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + + switch (conp->open_flag) { + + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + /* RDMA-write-to local memory area */ + if (!scratch_pad) { + scratch_pad = mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + dprintf("MPID_nem_ib_com_alloc,mmap=%p,len=%d\n", scratch_pad, sz); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(scratch_pad == (void *) -1, -1, + dprintf("failed to allocate buffer\n")); + dprintf("MPID_nem_ib_com_alloc,scratch_pad=%p\n", scratch_pad); + memset(scratch_pad, 0, sz); + } + conp->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] = scratch_pad; + conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_TO] = sz; + + conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO] = + MPID_nem_ib_com_reg_mr_fetch(conp->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO], + conp->icom_msize[MPID_NEM_IB_COM_SCRATCH_PAD_TO]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO], -1, + dprintf("ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags)); + +#ifdef HAVE_LIBDCFA + dprintf("MPID_nem_ib_com_alloc,fd=%d,rmem=%p\n", condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO]->buf); +#else + dprintf("MPID_nem_ib_com_alloc,fd=%d,rmem=%p\n", condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO]->addr); +#endif + dprintf("MPID_nem_ib_com_alloc,fd=%d,rkey=%08x\n", condesc, + conp->icom_mrlist[MPID_NEM_IB_COM_SCRATCH_PAD_TO]->rkey); + break; + default: + MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, + dprintf("MPID_nem_ib_com_alloc, invalid open_flag=%d\n", conp->open_flag)); + break; + } + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_close(int condesc) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + dprintf("MPID_nem_ib_com_close,condesc=%d\n", condesc); + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_nem_ib_com_clean(conp); + --maxcon; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + int ib_errno; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + if (conp->icom_connected == 1) { + ibcom_errno = -1; + goto fn_fail; + } + + struct ibv_qp_attr attr; + int flags; + + switch (conp->open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + /* Init QP */ + ib_errno = modify_qp_to_init(conp->icom_qp, conp->icom_port); + if (ib_errno) { + fprintf(stderr, "change QP state to INIT failed\n"); + ibcom_errno = ib_errno; + goto fn_fail; + } + /* Modify QP TO RTR status */ + ib_errno = + modify_qp_to_rtr(conp->icom_qp, remote_qpnum, remote_lid, remote_gid, conp->icom_port, + 0); + conp->remote_lid = remote_lid; /* for debug */ + if (ib_errno) { + fprintf(stderr, "failed to modify QP state to RTR\n"); + ibcom_errno = ib_errno; + goto fn_fail; + } + /* Modify QP TO RTS status */ + ib_errno = modify_qp_to_rts(conp->icom_qp); + if (ib_errno) { + fprintf(stderr, "failed to modify QP state to RTS\n"); + ibcom_errno = ib_errno; + goto fn_fail; + } + break; + case MPID_NEM_IB_COM_OPEN_UD: + /* INIT */ + memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_INIT; + attr.port_num = conp->icom_port; + attr.pkey_index = 0; + attr.qkey = MPID_NEM_IB_COM_QKEY; + flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY; + ib_errno = ibv_modify_qp(conp->icom_qp, &attr, flags); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_modify_qp")); + + /* RTR */ + memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_RTR; + flags = IBV_QP_STATE; + ib_errno = ibv_modify_qp(conp->icom_qp, &attr, flags); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_modify_qp")); + + /* RTS */ + memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_RTS; + attr.sq_psn = 0; + flags = IBV_QP_STATE | IBV_QP_SQ_PSN; + ib_errno = ibv_modify_qp(conp->icom_qp, &attr, flags); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_modify_qp")); + break; + } + conp->icom_connected = 1; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +#define MPID_NEM_IB_ENABLE_INLINE +int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, int sz_hdr, + void *data, int sz_data, int *copied) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + struct ibv_send_wr *bad_wr; + int ib_errno; + int num_sge; + + dprintf("MPID_nem_ib_com_isend,prefix=%p,sz_prefix=%d,hdr=%p,sz_hdr=%d,data=%p,sz_data=%d\n", prefix, + sz_prefix, hdr, sz_hdr, data, sz_data); + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + if (conp->icom_connected == 0) { + return -1; + } + int sz_data_pow2; + MPID_NEM_IB_SZ_DATA_POW2(sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data); + uint32_t sumsz = sz_data_pow2 + sizeof(MPID_nem_ib_tailmagic_t); + if (sz_data > 16000) { + //dprintf("MPID_nem_ib_com_isend,sz_data=%d,sz_data_pow2=%d,sz_max=%ld\n", sz_data, sz_data_pow2, MPID_NEM_IB_MAX_DATA_POW2); + } + + num_sge = 0; + + void *buf_from = + (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); + + MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf_from; + sz_hdrmagic->sz = sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t); + sz_hdrmagic->magic = MPID_NEM_IB_COM_MAGIC; + + /* memcpy hdr is needed because hdr resides in stack when sending close-VC command */ + /* memcpy is performed onto MPID_NEM_IB_COM_RDMAWR_FROM buffer */ + void *hdr_copy = (uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t); + memcpy(hdr_copy, prefix, sz_prefix); + memcpy((uint8_t *) hdr_copy + sz_prefix, hdr, sz_hdr); +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) sz_hdrmagic; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) sz_hdrmagic - + (uint64_t) conp-> + icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) sz_hdrmagic; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length = + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].lkey = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; + num_sge += 1; + + if (sz_data) { + //dprintf("MPID_nem_ib_com_isend,data=%p,sz_data=%d\n", data, sz_data); + struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) data; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = + mr_data->host_addr + ((uint64_t) data - (uint64_t) data); +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = (uint64_t) data; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length = sz_data; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].lkey = mr_data->lkey; + num_sge += 1; + } + + int sz_pad = sz_data_pow2 - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data); + MPID_nem_ib_tailmagic_t *tailmagic = + (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad); + tailmagic->magic = MPID_NEM_IB_COM_MAGIC; +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].mic_addr = + (uint64_t) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from + + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + + sz_hdr - + (uint64_t) conp-> + icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].addr = + (uint64_t) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_prefix + sz_hdr; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].length = sz_pad + sizeof(MPID_nem_ib_tailmagic_t); + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].sg_list[num_sge].lkey = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; + num_sge += 1; + dprintf("MPID_nem_ib_com_isend,sz_data=%d,pow2=%d,sz_pad=%d,num_sge=%d\n", sz_data, sz_data_pow2, sz_pad, + num_sge); + + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].num_sge = num_sge; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.remote_addr = + (uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); + /* rkey is defined in MPID_nem_ib_com_reg_mr_connect */ + + //dprintf("MPID_nem_ib_com_isend,condesc=%d,num_sge=%d,opcode=%08x,imm_data=%08x,wr_id=%016lx, raddr=%p, rkey=%08x\n", condesc, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].num_sge, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].opcode, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].imm_data, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr_id, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.remote_addr, conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey); + + /* other commands can executed RDMA-rd command */ + /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ +#if 0 + if (conp->after_rdma_rd) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags |= IBV_SEND_FENCE; + } +#endif +#ifdef MPID_NEM_IB_ENABLE_INLINE + if (sumsz <= conp->max_inline_data) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags |= IBV_SEND_INLINE; + *copied = 1; + } + else { + *copied = 0; + } +#endif +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", ib_errno)); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE], &bad_wr); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, + bad_wr)); +#endif +#ifdef MPID_NEM_IB_ENABLE_INLINE + if (sumsz <= conp->max_inline_data) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags &= ~IBV_SEND_INLINE; + } +#endif +#if 0 + if (conp->after_rdma_rd) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].send_flags &= ~IBV_SEND_FENCE; + conp->after_rdma_rd = 0; + } +#endif + + conp->sseq_num += 1; + assert(conp->sseq_num > 0); + conp->ncom += 1; + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, int sz_data) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + struct ibv_send_wr *bad_wr; + int ib_errno; + int sz_data_rem = sz_data; + int i; + struct ibv_mr *mr_data; + uint32_t sumsz = sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data + sizeof(MPID_nem_ib_tailmagic_t); + unsigned long tscs, tsce; + + dprintf("MPID_nem_ib_com_isend_chain,enter\n"); + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->icom_connected == 0, -1, + printf("MPID_nem_ib_com_isend_chain,icom_connected==0\n")); + + void *buf_from = + (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); + + /* make a tail-magic position is in a fixed set */ + int sz_data_pow2; + MPID_NEM_IB_SZ_DATA_POW2(sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data); + + /* let the last command icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAIN-1] which has IBV_WR_RDMA_WRITE_WITH_IMM */ + int s = + MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - (sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data_pow2 + + sizeof(MPID_nem_ib_tailmagic_t) + MPID_NEM_IB_COM_INLINE_DATA - 1) / MPID_NEM_IB_COM_INLINE_DATA; + MPID_NEM_IB_COM_ERR_CHKANDJUMP((sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr + sz_data_pow2) % 4 != 0, -1, + printf("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(s < 0 || + s >= MPID_NEM_IB_COM_SMT_INLINE_NCHAIN, -1, printf("MPID_nem_ib_com_isend_chain,s\n")); + dprintf("MPID_nem_ib_com_isend_chain,sz_hdr=%d,sz_data=%d,s=%d\n", sz_hdr, sz_data, s); + + for (i = s; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { + + //tscs = MPID_nem_ib_rdtsc(); + int sz_used = 0; /* how much of the payload of a IB packet is used? */ + int num_sge = 0; + if (i == s) { + MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf_from; + sz_hdrmagic->sz = sumsz; + sz_hdrmagic->magic = MPID_NEM_IB_COM_MAGIC; + memcpy((uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t), hdr, sz_hdr); +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = + (uint64_t) buf_from; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - + (uint64_t) conp-> + icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + (uint64_t) buf_from; +#endif + buf_from = (uint8_t *) buf_from + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr; + sz_used += sizeof(MPID_nem_ib_sz_hdrmagic_t) + sz_hdr; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; + num_sge += 1; + dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d\n", i, sz_used); + } + //tsce = MPID_nem_ib_rdtsc(); printf("0,%ld\n", tsce-tscs); + + //tscs = MPID_nem_ib_rdtsc(); + if (sz_data_rem > 0) { +#ifdef HAVE_LIBDCFA +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + (uint64_t) data + sz_data - sz_data_rem; +#endif + int sz_data_red = + sz_used + sz_data_rem + sizeof(MPID_nem_ib_tailmagic_t) <= + MPID_NEM_IB_COM_INLINE_DATA ? sz_data_rem : sz_data_rem <= + MPID_NEM_IB_COM_INLINE_DATA - sz_used ? sz_data_rem : MPID_NEM_IB_COM_INLINE_DATA - sz_used; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_data_red; + sz_used += sz_data_red; + sz_data_rem -= sz_data_red; + MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem < 0, -1, printf("MPID_nem_ib_com_isend_chain,sz_data_rem\n")); + + if (i == s) { + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, printf("MPID_nem_ib_com_isend_chain,sz_data==0\n")); + mr_data = MPID_nem_ib_com_reg_mr_fetch(data, sz_data); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, printf("MPID_nem_ib_com_isend,ibv_reg_mr_fetch failed\n")); + } +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = + (uint64_t) data + sz_data - sz_data_rem; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + mr_data->host_addr + ((uint64_t) data + sz_data - sz_data_rem - (uint64_t) data); +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = mr_data->lkey; + num_sge += 1; + dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_used=%d,sz_data_rem=%d\n", i, sz_used, sz_data_rem); + } + else { /* tailmagic only packet is being generated */ + + } + //tsce = MPID_nem_ib_rdtsc(); printf("1,%ld\n", tsce-tscs); + + //tscs = MPID_nem_ib_rdtsc(); + if (i == MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */ + int sz_pad = sz_data_pow2 - sz_data; + MPID_nem_ib_tailmagic_t *tailmagic = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf_from + sz_pad); + tailmagic->magic = MPID_NEM_IB_COM_MAGIC; +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = + (uint64_t) buf_from; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - + (uint64_t) conp-> + icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + (uint64_t) buf_from; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = + sz_pad + sizeof(MPID_nem_ib_tailmagic_t); + sz_used += sz_pad + sizeof(MPID_nem_ib_tailmagic_t); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_data_rem != 0, -1, printf("MPID_nem_ib_com_isend_chain, sz_data_rem\n")); + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; + num_sge += 1; + + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].imm_data = conp->sseq_num; + dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d,num_sge=%d\n", i, sz_pad, sz_used, + num_sge); + } + else if (MPID_NEM_IB_COM_INLINE_DATA - sz_used > 0) { /* data fell short of the packet, so pad */ + MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, + printf + ("MPID_nem_ib_com_isend_chain,tail-magic gets over packet-boundary\n")); + int sz_pad = MPID_NEM_IB_COM_INLINE_DATA - sz_used; +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = + (uint64_t) buf_from; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->host_addr + ((uint64_t) buf_from - + (uint64_t) conp-> + icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM]); +#else + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = + (uint64_t) buf_from; +#endif + buf_from = (uint8_t *) buf_from + sz_pad; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad; + sz_used += sz_pad; + MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1, + printf("MPID_nem_ib_com_isend_chain, sz_used\n")); + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].lkey = + conp->icom_mrlist[MPID_NEM_IB_COM_RDMAWR_FROM]->lkey; + num_sge += 1; + dprintf("MPID_nem_ib_com_isend_chain,i=%d,sz_pad=%d,sz_used=%d\n", i, sz_pad, sz_used); + } + else { /* packet is full with data */ + MPID_NEM_IB_COM_ERR_CHKANDJUMP(sz_used != MPID_NEM_IB_COM_INLINE_DATA, -1, + printf("MPID_nem_ib_com_isend_chain, sz_used\n")); + } + //tsce = MPID_nem_ib_rdtsc(); printf("2,%ld\n", tsce-tscs); + + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].num_sge = num_sge; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.remote_addr = + (uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG) + MPID_NEM_IB_COM_INLINE_DATA * (i - + s); + } +#if 0 + if (conp->after_rdma_rd) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s].send_flags |= IBV_SEND_FENCE; + } +#endif +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s]); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s], &bad_wr); +#endif +#if 0 + if (i == 0 && conp->after_rdma_rd) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + s].send_flags &= ~IBV_SEND_FENCE; + conp->after_rdma_rd = 0; + } +#endif +#ifdef HAVE_LIBDCFA + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d\n", ib_errno)); +#else + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_isend, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, + bad_wr)); +#endif + conp->ncom += (MPID_NEM_IB_COM_SMT_INLINE_NCHAIN - s); + conp->sseq_num += 1; + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id) +{ + + MPID_nem_ib_com_t *conp; + int ib_errno; + int ibcom_errno = 0; + struct ibv_recv_wr *bad_wr; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + // if (conp->icom_connected == 0) { return -1; } + + //dprintf("MPID_nem_ib_com_irecv,condesc=%d,wr_id=%016lx\n", condesc, wr_id); + + conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER].wr_id = wr_id; +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER]); +#else + ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_RDMAWR_RESPONDER], &bad_wr); +#endif + if (ib_errno) { +#ifdef HAVE_LIBDCFA + fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d\n", ib_errno); +#else + fprintf(stderr, "MPID_nem_ib_com_irecv: failed to post receive, ib_errno=%d,bad_wr=%p\n", ib_errno, + bad_wr); +#endif + ibcom_errno = ib_errno; + goto fn_fail; + } + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, uint32_t remote_qpn, + uint32_t imm_data, uint64_t wr_id) +{ + MPID_nem_ib_com_t *conp; + struct ibv_send_wr *bad_wr; + int ibcom_errno = 0, ib_errno; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + +#ifdef HAVE_LIBDCFA + MPID_NEM_IB_COM_ERR_CHKANDJUMP(1, -1, + dprintf + ("MPID_nem_ib_com_udsend not supported by DCFA because DCFA doesn't have ibv_create_ah\n")); +#else + /* prepare ibv_ah_attr */ + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].dlid = remote_lid; +#if 0 + conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].grh.dgid = *remote_gid; +#endif + + /* prepare ibv_ah */ + struct ibv_ah *ah; + ah = ibv_create_ah(ib_pd, &conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!ah, -1, dprintf("ibv_crate_ah\n")); + + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].wr.ud.ah = ah; + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].wr.ud.remote_qpn = remote_qpn; + /* qkey is defined in open */ + + //dprintf("lid=%04x\n", conp->icom_ah_attr[MPID_NEM_IB_COM_UD_INITIATOR].dlid); + //dprintf("qpn=%08x\n", conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].wr.ud.remote_qpn); + + /* recv doesn't know the length, so we can't optimize it */ + // conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].sg_list[0].length = length; + + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].imm_data = imm_data; + +#if 0 + if (length <= qpinfo->max_inline_data) { + conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR].send_flags |= IBV_SEND_INLINE; + } +#endif + +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR]); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_UD_INITIATOR], &bad_wr); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, perror("ibv_post_send")); +#endif /* DCFA */ + + conp->ncom += 1; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_udrecv(int condesc) +{ + MPID_nem_ib_com_t *conp; + struct ibv_recv_wr *bad_wr; + int ibcom_errno = 0, ib_errno; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + + /* Create RR */ + conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER].wr_id = 0; + + /* Post RR to RQ */ +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER]); +#else + ib_errno = ibv_post_recv(conp->icom_qp, &conp->icom_rr[MPID_NEM_IB_COM_UD_RESPONDER], &bad_wr); +#endif + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("ibv_post_recv ib_errno=%d\n", ib_errno)); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + struct ibv_send_wr *bad_wr; + int ib_errno; + int num_sge; + + dprintf("MPID_nem_ib_com_lrecv,enter,raddr=%p,sz_data=%d,laddr=%p\n", raddr, sz_data, laddr); + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("MPID_nem_ib_com_lrecv,not connected\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("MPID_nem_ib_com_lrecv,sz_data==0\n")); + + num_sge = 0; + + /* register memory area containing data */ + struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz_data); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_lrecv,ibv_reg_mr_fetch failed\n")); + + /* Erase magic, super bug!! */ + //((MPID_nem_ib_tailmagic_t*)(laddr + sz_data - sizeof(MPID_nem_ib_tailmagic_t)))->magic = 0; +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list[num_sge].mic_addr = (uint64_t) laddr; + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list[num_sge].addr = + mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); +#else + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list[num_sge].addr = (uint64_t) laddr; +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list[num_sge].length = sz_data; + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].sg_list[num_sge].lkey = mr_data->lkey; + num_sge += 1; + + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].num_sge = num_sge; + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].wr.rdma.remote_addr = (uint64_t) raddr; + conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR].wr.rdma.rkey = rkey; + +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d\n", ib_errno)); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_INITIATOR], &bad_wr); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_lrecv, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, + bad_wr)); +#endif + + /* other commands can be executed before RDMA-rd command */ + /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ +#if 0 + conp->after_rdma_rd = 1; +#endif + conp->ncom += 1; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +/* use the same QP as isend */ +int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + struct ibv_send_wr *bad_wr; + int ib_errno; + int num_sge; + + dprintf("MPID_nem_ib_com_put_lmt,enter,sz_data=%d,laddr=%p\n", sz_data, laddr); + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, dprintf("MPID_nem_ib_com_put_lmt,not connected\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz_data, -1, dprintf("MPID_nem_ib_com_put_lmt,sz_data==0\n")); + + num_sge = 0; + + /* register memory area containing data */ + struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz_data); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_lmt,ibv_reg_mr_fetch failed\n")); + +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].mic_addr = (uint64_t) laddr; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].addr = + mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); +#else + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].addr = (uint64_t) laddr; +#endif + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].length = sz_data; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].sg_list[num_sge].lkey = mr_data->lkey; + num_sge += 1; + + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].num_sge = num_sge; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr.rdma.remote_addr = (uint64_t) raddr; + conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT].wr.rdma.rkey = rkey; + +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d\n", ib_errno)); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_LMT_PUT], &bad_wr); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_put_lmt, ibv_post_send, rc=%d, bad_wr=%p\n", ib_errno, + bad_wr)); +#endif + + conp->ncom += 1; + dprintf("MPID_nem_ib_com_put_lmt,exit\n"); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + struct ibv_send_wr *bad_wr; + int ib_errno; + + dprintf("MPID_nem_ib_com_put_scratch_pad,enter,wr_id=%llx,offset=%llx,sz=%d,laddr=%p\n", + (unsigned long long) wr_id, (unsigned long long) offset, sz, laddr); + dprintf("MPID_nem_ib_com_put_scratch_pad,data=%08x\n", *((uint32_t *) laddr)); + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->open_flag != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag)); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,sz==0\n")); + + /* register memory area containing data */ + struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n")); + dprintf("MPID_nem_ib_com_put_scratch_pad,"); + +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].mic_addr = (uint64_t) laddr; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = + mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); +#else + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = (uint64_t) laddr; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].length = sz; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].lkey = mr_data->lkey; + + /* num_sge is defined in MPID_nem_ib_com_open */ + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr = + (uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + offset; + /* rkey is defined in MPID_nem_ib_com_reg_mr_connect */ + + dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n", + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr); + +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno)); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], &bad_wr); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); +#endif + + conp->ncom_scratch_pad += 1; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +#ifdef MPID_NEM_IB_ONDEMAND +int MPID_nem_ib_com_cas_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, uint64_t compare, + uint64_t swap) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + struct ibv_send_wr *bad_wr; + int ib_errno; + + dprintf("MPID_nem_ib_com_put_scratch_pad,enter,wr_id=%llx,offset=%llx,sz=%d,laddr=%p\n", + (unsigned long long) wr_id, (unsigned long long) offset, sz, laddr); + dprintf("MPID_nem_ib_com_put_scratch_pad,data=%08x\n", *((uint32_t *) laddr)); + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(conp->open_flag != MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad,invalid open_flag=%d\n", conp->open_flag)); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!conp->icom_connected, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad,not connected\n")); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!sz, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,sz==0\n")); + + /* register memory area containing data */ + struct ibv_mr *mr_data = MPID_nem_ib_com_reg_mr_fetch(laddr, sz); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(!mr_data, -1, dprintf("MPID_nem_ib_com_put_scratch_pad,ibv_reg_mr_fetch failed\n")); + dprintf("MPID_nem_ib_com_put_scratch_pad,"); + +#ifdef HAVE_LIBDCFA + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].mic_addr = (uint64_t) laddr; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = + mr_data->host_addr + ((uint64_t) laddr - (uint64_t) laddr); +#else + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].addr = (uint64_t) laddr; +#endif + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].length = sz; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].sg_list[0].lkey = mr_data->lkey; + + /* num_sge is defined in MPID_nem_ib_com_open */ + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr_id = wr_id; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.atomic.remote_addr = + (uint64_t) conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + offset; + /* rkey is defined in MPID_nem_ib_com_reg_mr_connect */ + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.atomic.compare_add = compare; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.atomic.swap = swap; + + dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n", + (unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.remote_addr); + +#ifdef HAVE_LIBDCFA + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d\n", ib_errno)); +#else + ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR], &bad_wr); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(ib_errno, -1, + dprintf("MPID_nem_ib_com_put_scratch_pad, ibv_post_send, rc=%d, bad_wr=%p\n", + ib_errno, bad_wr)); +#endif + + conp->ncom_scratch_pad += 1; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} +#endif + +/* poll completion queue */ +int MPID_nem_ib_com_poll_cq(int which_cq, struct ibv_wc *wc, int *result) +{ + int ibcom_errno = 0; + + switch (which_cq) { + case MPID_NEM_IB_COM_RC_SHARED_RCQ: + *result = ibv_poll_cq(MPID_nem_ib_rc_shared_rcq, 1, wc); + break; + case MPID_NEM_IB_COM_RC_SHARED_SCQ: + *result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, 1, wc); + break; + case MPID_NEM_IB_COM_UD_SHARED_RCQ: + *result = ibv_poll_cq(MPID_nem_ib_ud_shared_rcq, 1, wc); + break; + case MPID_NEM_IB_COM_UD_SHARED_SCQ: + *result = ibv_poll_cq(MPID_nem_ib_ud_shared_scq, 1, wc); + break; + } + + if (*result < 0) { + dprintf("MPID_nem_ib_com_poll_cq,status=%08x,vendor_err=%08x,len=%d,opcode=%08x,wr_id=%016lx\n", + wc->status, wc->vendor_err, wc->byte_len, wc->opcode, wc->wr_id); + ibcom_errno = *result; + goto fn_fail; + } + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_reg_mr_connect(int condesc, void *rmem, int rkey) +{ + int ibcom_errno = 0; + MPID_nem_ib_com_t *conp; + int i; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + switch (conp->open_flag) { + case MPID_NEM_IB_COM_OPEN_RC: + case MPID_NEM_IB_COM_OPEN_RC_LMT_PUT: + conp->icom_rmem[MPID_NEM_IB_COM_RDMAWR_TO] = rmem; + conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO] = rkey; + conp->icom_sr[MPID_NEM_IB_COM_SMT_NOINLINE].wr.rdma.rkey = conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO]; + for (i = 0; i < MPID_NEM_IB_COM_SMT_INLINE_NCHAIN; i++) { + conp->icom_sr[MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 + i].wr.rdma.rkey = + conp->icom_rkey[MPID_NEM_IB_COM_RDMAWR_TO]; + } + break; + + case MPID_NEM_IB_COM_OPEN_SCRATCH_PAD: + conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] = rmem; + conp->icom_rkey[MPID_NEM_IB_COM_SCRATCH_PAD_TO] = rkey; + conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.rkey = + conp->icom_rkey[MPID_NEM_IB_COM_SCRATCH_PAD_TO]; + break; + + default: + dprintf("invalid open_flag=%d\n", conp->open_flag); + break; + } + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_get_info_conn(int condesc, int key, void *out, uint32_t out_len) +{ + int ibcom_errno = 0; + MPID_nem_ib_com_t *conp; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + + switch (key) { + case MPID_NEM_IB_COM_INFOKEY_QP_QPN: + memcpy(out, &conp->icom_qp->qp_num, out_len); + break; + case MPID_NEM_IB_COM_INFOKEY_PORT_LID: +#ifdef HAVE_LIBDCFA + dprintf("MPID_nem_ib_com_get_info_conn,lid=%04x\n", ib_ctx->lid); + memcpy(out, &ib_ctx->lid, out_len); +#else + dprintf("MPID_nem_ib_com_get_info_conn,lid=%04x\n", conp->icom_pattr.lid); + memcpy(out, &conp->icom_pattr.lid, out_len); +#endif + break; + case MPID_NEM_IB_COM_INFOKEY_PORT_GID: + memcpy(out, &conp->icom_gid, out_len); + break; + case MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ:{ +#ifdef HAVE_LIBDCFA + uint32_t max_msg_sz = 1073741824; /* ConnectX-3 */ + memcpy(out, &max_msg_sz, out_len); +#else + memcpy(out, &conp->icom_pattr.max_msg_sz, out_len); +#endif + break; + } + default: + ibcom_errno = -1; + break; + } + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_get_info_mr(int condesc, int memid, int key, void *out, int out_len) +{ + int ibcom_errno = 0; + MPID_nem_ib_com_t *conp; + struct ibv_mr *mr; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + MPID_NEM_IB_COM_ERR_CHKANDJUMP(memid >= conp->icom_mrlen, -1, + dprintf("MPID_nem_ib_com_get_info_mr,wrong mem_id=%d\n", memid)); + mr = conp->icom_mrlist[memid]; + + switch (key) { + case MPID_NEM_IB_COM_INFOKEY_MR_ADDR: +#ifdef HAVE_LIBDCFA + /* host_addr is created by ibv_reg_mr in MPID_nem_ib_com_open, */ + /* ib_init read this host-addr, put it into KVS, the counter-party read it through KVS */ + memcpy(out, &mr->host_addr, out_len); +#else + memcpy(out, &mr->addr, out_len); +#endif + break; + case MPID_NEM_IB_COM_INFOKEY_MR_LENGTH:{ +#ifdef HAVE_LIBDCFA + assert(out_len == sizeof(size_t)); + size_t length = mr->size; /* type of mr->size is int */ + memcpy(out, &length, out_len); +#else + memcpy(out, &mr->length, out_len); +#endif + break; + } + case MPID_NEM_IB_COM_INFOKEY_MR_RKEY: + memcpy(out, &mr->rkey, out_len); + break; + default: + dprintf("MPID_nem_ib_com_get_info_mr,unknown key=%d\n", key); + ibcom_errno = -1; + break; + } + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_mem_rdmawr_from(int condesc, void **out) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *out = + (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_FROM] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * (conp->sseq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_mem_rdmawr_to(int condesc, int seq_num, void **out) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *out = (uint8_t *) conp->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZSEG * (seq_num % MPID_NEM_IB_COM_RDMABUF_NSEG); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_mem_udwr_from(int condesc, void **out) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *out = conp->icom_mem[MPID_NEM_IB_COM_UDWR_FROM]; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_mem_udwr_to(int condesc, void **out) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *out = conp->icom_mem[MPID_NEM_IB_COM_UDWR_TO]; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_sseq_num_get(int condesc, int *seq_num) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *seq_num = conp->sseq_num; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_lsr_seq_num_tail_get(int condesc, int **seq_num) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *seq_num = &(conp->lsr_seq_num_tail); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_rsr_seq_num_tail_get(int condesc, int **seq_num) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *seq_num = &(conp->rsr_seq_num_tail); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get(int condesc, int **seq_num) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *seq_num = &(conp->rsr_seq_num_tail_last_sent); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(int condesc, int *notify_rate) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + + switch (conp->rdmabuf_occupancy_notify_lstate) { + case MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW: + *notify_rate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW; + break; + case MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW: + *notify_rate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW; + break; + default: + ibcom_errno = -1; + goto fn_fail; + break; + } + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(int condesc, int **rstate) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *rstate = &(conp->rdmabuf_occupancy_notify_rstate); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get(int condesc, int **lstate) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *lstate = &(conp->rdmabuf_occupancy_notify_lstate); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_obtain_pointer(int condesc, MPID_nem_ib_com_t ** MPID_nem_ib_com) +{ + MPID_nem_ib_com_t *conp; + int ibcom_errno = 0; + + MPID_NEM_IB_RANGE_CHECK_WITH_ERROR(condesc, conp); + *MPID_nem_ib_com = conp; + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +static void MPID_nem_ib_comShow(int condesc) +{ + MPID_nem_ib_com_t *conp; + uint8_t *p; + int i; + + MPID_NEM_IB_RANGE_CHECK(condesc, conp); + fprintf(stdout, "qp_num = %d\n", conp->icom_qp->qp_num); +#ifdef HAVE_LIBDCFA + fprintf(stdout, "lid = %d\n", ib_ctx->lid); +#else + fprintf(stdout, "lid = %d\n", conp->icom_pattr.lid); +#endif + p = (uint8_t *) & conp->icom_gid; + fprintf(stdout, "gid = %02x", p[0]); + for (i = 1; i < 16; i++) { + fprintf(stdout, ":%02x", p[i]); + } + fprintf(stdout, "\n"); +} + +static char *strerror_tbl[] = { + [0] = "zero", + [1] = "one", + [2] = "two", + [3] = "three", +}; + +char *MPID_nem_ib_com_strerror(int errno) +{ + char *r; + if (-errno > 3) { + r = malloc(256); + sprintf(r, "%d", -errno); + goto fn_exit; + } + else { + r = strerror_tbl[-errno]; + } + fn_exit: + return r; + //fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_reg_mr(void *addr, int len, struct ibv_mr **mr) +{ + int ibcom_errno = 0; + dprintf("MPID_nem_ib_com_reg_mr,addr=%p,len=%d,mr=%p\n", addr, len, mr); + + *mr = + ibv_reg_mr(ib_pd, addr, len, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ); + + MPID_NEM_IB_COM_ERR_CHKANDJUMP(*mr == 0, -1, dprintf("MPID_nem_ib_com_reg_mr,cannot register memory\n")); + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} + +int MPID_nem_ib_com_dereg_mr(struct ibv_mr *mr) +{ + int i; + int ib_errno; + int ibcom_errno = 0; + + if (!mr) { + goto fn_exit; + } + + ib_errno = ibv_dereg_mr(mr); + if (ib_errno < 0) { + fprintf(stderr, "cannot deregister memory\n"); + goto fn_fail; + } +#ifdef HAVE_LIBDCFA + dprintf("MPID_nem_ib_com_dereg_mr, addr=%p\n", mr->buf); +#else + dprintf("MPID_nem_ib_com_dereg_mr, addr=%p\n", mr->addr); +#endif + + fn_exit: + return ibcom_errno; + fn_fail: + goto fn_exit; +} diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h new file mode 100644 index 0000000..fcd1ea3 --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h @@ -0,0 +1,417 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2012 NEC Corporation + * Author: Masamichi Takagi + * (C) 2012 Oct 14 Yutaka Ishikawa, ishikawa(a)is.s.u-tokyo.ac.jp + * See COPYRIGHT in top-level directory. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <stdint.h> +#include <sys/types.h> +#include "mpid_nem_impl.h" + +#ifdef HAVE_LIBDCFA +#include "dcfa.h" + +/* +*** diff -p verbs.h dcfa.h (structures) +same name, same fields + struct ibv_device { }; + struct ibv_context { }; + struct ibv_pd { }; + struct ibv_ah_attr { }; + +same name, different fields + struct ibv_qp_init_attr { +- void *qp_context; +- struct ibv_xrc_domain *xrc_domain; +}; + + struct ibv_mr { +- void *addr; ++ void *buf; ++ uint64_t host_addr; +- size_t length; ++ int size; +- uint32_t handle; ++ uint64_t handle; ++ int flag; 1: offload +- uint32_t lkey; ++ int lkey; +- uint32_t rkey; ++ int rkey; +}; + + struct ibv_qp { ++ struct mlx4_buf buf; ++ int max_inline_data; ++ int buf_size; + ++ uint32_t doorbell_qpn; ++ uint32_t sq_signal_bits; ++ int sq_spare_wqes; ++ struct mlx4_wq sq; + ++ uint32_t *db; // doorbell addr for post recv ++ struct mlx4_wq rq; ++ ibmic_qp_conn_info_t remote_qp_info; + +- uint32_t handle; ++ uint64_t handle; + +- struct ibv_context *context; +- void *qp_context; +- uint32_t events_completed; +- struct ibv_xrc_domain *xrc_domain; +- pthread_mutex_t mutex; +- pthread_cond_t cond; +}; + + struct ibv_cq { +- struct ibv_comp_channel *channel; +- void *cq_context; +- uint32_t handle; +- uint32_t comp_events_completed; +- uint32_t async_events_completed; + +- pthread_mutex_t mutex; +- pthread_cond_t cond; + ++ struct mlx4_buf buf; ++ uint32_t cons_index; ++ uint32_t wait_index; ++ uint32_t *set_ci_db; ++ uint32_t *arm_db; ++ int arm_sn; ++ int cqe_size; ++ uint64_t handle; +}; + + struct ibv_wc { +- uint32_t src_qp; +- uint16_t pkey_index; +- uint16_t slid; +- uint8_t sl; +- uint8_t dlid_path_bits; +}; + + struct ibv_send_wr { +- struct ibv_sge *sg_list; ++ struct ibv_sge sg_list[WR_SG_NUM]; ++ uint64_t addr; ++ uint32_t length; ++ uint32_t lkey; + }; + + struct ibv_recv_wr { +- struct ibv_sge *sg_list; ++ struct ibv_sge sg_list[WR_SG_NUM]; + }; + + struct ibv_sge { ++ uint64_t mic_addr; // buffer address on mic + }; + +non-existent +- struct ibv_port_attr { }; + + +*** diff -p verbs.h dcfa.h (functions) + +same name, same arguments + ibv_get_device_list + ibv_open_device + ibv_close_device + ibv_free_device_list + ibv_alloc_pd + ibv_dealloc_pd + ibv_create_qp + ibv_destroy_qp + ibv_reg_mr + ibv_dereg_mr + ibv_destroy_cq + ibv_poll_cq + ibv_modify_qp + +same name, different arguments +- int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr) ++ int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr); + +- int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) ++ int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr); + +- struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector); ++ struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe_max); + +non-existent +- ibv_get_device_name +- ibv_query_port +- ibv_query_gid +- ibv_create_ah +struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); +*/ + +#else +/* Original Infiniband */ +#include <infiniband/verbs.h> +#endif + +static inline unsigned long long MPID_nem_ib_rdtsc_cpuid(void) +{ + unsigned int lo, hi; + __asm__ __volatile__(// serialize + "xorl %%eax,%%eax \n cpuid":::"%rax", "%rbx", "%rcx", "%rdx"); + __asm__ __volatile__("rdtsc":"=a"(lo), "=d"(hi)); + return (unsigned long long) hi << 32 | lo; +} + +extern struct ibv_cq *MPID_nem_ib_rc_shared_scq; +extern struct ibv_cq *MPID_nem_ib_rc_shared_scq_lmt_put; +extern struct ibv_cq *MPID_nem_ib_rc_shared_scq_scratch_pad; +extern struct ibv_cq *MPID_nem_ib_ud_shared_rcq; + +#define MPID_NEM_IB_COM_SIZE 2048 /* one process uses 2-4 fds */ +#define MPID_NEM_IB_COM_INLINE_DATA (512-64) /* experimented max is 884 */ /* this is lower bound and more than this value is set. the more this value is, the more the actual value set is. you need to check it */ + +#define MPID_NEM_IB_COM_MAX_SQ_CAPACITY (256/1) +#define MPID_NEM_IB_COM_MAX_RQ_CAPACITY ((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)+16) /* We pre-post_recv MPID_NEM_IB_COM_MAX_SQ_CAPACITY of commands */ +#define MPID_NEM_IB_COM_MAX_SGE_CAPACITY (32/2) /* maximum for ConnectX-3 looks like 32 */ +#define MPID_NEM_IB_COM_MAX_CQ_CAPACITY MPID_NEM_IB_COM_MAX_RQ_CAPACITY +#define MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>1)) /* drain when reaching this amount */ +#define MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN (((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>2)+((MPID_NEM_IB_COM_MAX_SQ_CAPACITY)>>1)) /* drain when reaching this amount */ +#define MPID_NEM_IB_COM_AMT_CQ_DRAIN ((MPID_NEM_IB_COM_MAX_CQ_CAPACITY)>>2) /* drain this amount */ +#define MPID_NEM_IB_COM_MAX_RD_ATOMIC 4 + +#define MPID_NEM_IB_COM_MAX_TRIES 1 +#define MPID_NEM_IB_COM_SCQ_FLG 1 +#define MPID_NEM_IB_COM_RCQ_FLG 2 + +#define MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ 100 +#define MPID_NEM_IB_COM_INFOKEY_MR_ADDR 200 +#define MPID_NEM_IB_COM_INFOKEY_MR_LENGTH 201 +#define MPID_NEM_IB_COM_INFOKEY_MR_RKEY 202 +#define MPID_NEM_IB_COM_INFOKEY_QP_QPN 300 +#define MPID_NEM_IB_COM_INFOKEY_PORT_LID 400 +#define MPID_NEM_IB_COM_INFOKEY_PORT_GID 401 + + +/* buffers */ +#define MPID_NEM_IB_COM_NBUF_RDMA 2 /* number of <addr, sz, lkey, rkey> */ +#define MPID_NEM_IB_COM_RDMAWR_FROM 0 /* index to RDMA-write-from buffer */ +#define MPID_NEM_IB_COM_RDMAWR_TO 1 /* index to RDMA-write-to buffer */ +/* assuming that the unit (32768) is equals to eager-RDMA-write threashold + assuming that the multiplier (256) is + equals to max number of outstanding eager-RDMA-write transactions */ +#define MPID_NEM_IB_COM_RDMABUF_SZSEG (16384/4) //(16384+8+40+1) /* this size minus magics and headers must be 2^n because data might grow to the next 2^m boundary, see ib_impl.h, ib_com.c, src/mpid/ch3/src/mpid_isend.c */ +#define MPID_NEM_IB_COM_RDMABUF_SZ ((MPID_NEM_IB_COM_RDMABUF_SZSEG) * 16) /* (32768 * 256) */ +#define MPID_NEM_IB_COM_RDMABUF_NSEG ((MPID_NEM_IB_COM_RDMABUF_SZ) / (MPID_NEM_IB_COM_RDMABUF_SZSEG)) +#define MPID_NEM_IB_COM_SMT_INLINE_NCHAIN 8 /* maximum number of chained inline-send commands */ +#define MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>1)+((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) +#define MPID_NEM_IB_COM_RDMABUF_LOW_WATER_MARK (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW 1 +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW 2 +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW /*1*/(((MPID_NEM_IB_COM_RDMABUF_NSEG)>>4) == 0 ? 1 : ((MPID_NEM_IB_COM_RDMABUF_NSEG)>>4)) +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((MPID_NEM_IB_COM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */ +#define MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */ + +#define MPID_NEM_IB_COM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */ +#define MPID_NEM_IB_COM_UDWR_FROM 0 /* index to UD-write-from buffer */ +#define MPID_NEM_IB_COM_UDWR_TO 1 /* index to UD-write-to buffer */ +#define MPID_NEM_IB_COM_UDBUF_SZ (128 * 8192) /* supporting 100K ranks with 10 rounds */ +#define MPID_NEM_IB_COM_UDBUF_SZSEG (128) +#define MPID_NEM_IB_COM_UDBUF_NSEG (MPID_NEM_IB_COM_UDBUF_SZ / MPID_NEM_IB_COM_UDBUF_SZSEG) + +#define MPID_NEM_IB_COM_NBUF_SCRATCH_PAD 1 /* number of <addr, sz, lkey, rkey> */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_TO 0 /* index to RDMA-write-to buffer */ + +/* send command templates */ +#define MPID_NEM_IB_COM_RC_SR_NTEMPLATE (8+1+2) /* number of request templates, 8 for inline-chained-smt, 1 for smt, 1 for lmt */ +#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED0 0 /* index to it */ +#define MPID_NEM_IB_COM_SMT_INLINE_CHAINED7 7 +#define MPID_NEM_IB_COM_SMT_NOINLINE 8 +#define MPID_NEM_IB_COM_LMT_INITIATOR 9 /* FIXME: bad naming */ + +#define MPID_NEM_IB_COM_RC_SR_LMT_PUT_NTEMPLATE MPID_NEM_IB_COM_RC_SR_NTEMPLATE /* FIXME: TEMPLATE named MPID_NEM_IB_COM_RC_SR shares MPID_NEM_IB_COM_LMT_PUT */ +#define MPID_NEM_IB_COM_LMT_PUT 10 + +/* recv command templates */ +#define MPID_NEM_IB_COM_RC_RR_NTEMPLATE 1 /* 1 for smt, */ +#define MPID_NEM_IB_COM_RDMAWR_RESPONDER 0 /* index to recv request template */ + +/* sge template */ +#define MPID_NEM_IB_COM_SMT_INLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ +#define MPID_NEM_IB_COM_SMT_NOINLINE_INITIATOR_NSGE 4 /* MPI header, (sz;magic), data x1, magic */ +#define MPID_NEM_IB_COM_LMT_INITIATOR_NSGE 1 /* data x1 */ +#define MPID_NEM_IB_COM_LMT_PUT_NSGE 1 /* data x1 */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR_NSGE 1 /* QP state */ + +#define MPID_NEM_IB_COM_UD_SR_NTEMPLATE 1 +#define MPID_NEM_IB_COM_UD_RR_NTEMPLATE 1 +#define MPID_NEM_IB_COM_UD_INITIATOR 0 /* index to send request template */ +#define MPID_NEM_IB_COM_UD_RESPONDER 0 /* index to recv request template */ + +#define MPID_NEM_IB_COM_SCRATCH_PAD_SR_NTEMPLATE 2 +#define MPID_NEM_IB_COM_SCRATCH_PAD_RR_NTEMPLATE 1 +#define MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR 0 /* index to send request template */ +#define MPID_NEM_IB_COM_SCRATCH_PAD_CAS 1 +#define MPID_NEM_IB_COM_SCRATCH_PAD_RESPONDER 0 /* index to recv request template */ + + +typedef struct MPID_nem_ib_com { + short icom_used; + short icom_connected; + int icom_port; +#ifdef HAVE_LIBDCFA +#else + struct ibv_port_attr icom_pattr; /* IB port attributes */ +#endif + struct ibv_qp *icom_qp; + struct ibv_cq *icom_scq; + struct ibv_cq *icom_rcq; + struct ibv_mr **icom_mrlist; + int icom_mrlen; + union ibv_gid icom_gid; + void **icom_mem; /* 0: send 1: recv 2..: rdma */ + int *icom_msize; /* 0: send 1: recv 2..: rdma */ + struct ibv_send_wr *icom_sr; + struct ibv_ah_attr *icom_ah_attr; + struct ibv_recv_wr *icom_rr; + void **icom_rmem; + int *icom_rkey; + size_t *icom_rsize; + int sseq_num; + int rsr_seq_num_poll; + int rsr_seq_num_tail; /* occupation status of remote Send Request (SR) queue (it covers occupation status of local RDMA-wr-to buffer) */ + int rsr_seq_num_tail_last_sent; /* latest one sent to remote rank */ + int lsr_seq_num_tail; /* occupation status of local Send Request (SR) queue */ + int lsr_seq_num_tail_last_requested; /* value when lmt_start_send issued req_seq_num */ + int rdmabuf_occupancy_notify_rstate, rdmabuf_occupancy_notify_lstate; + int ncom, ncom_lmt_put, ncom_scratch_pad; /* number of entries in the command queue */ + + uint32_t max_inline_data; /* actual value obtained after ibv_create_qp */ + uint32_t max_send_wr; + uint32_t max_recv_wr; + + uint32_t open_flag; /* MPID_NEM_IB_COM_OPEN_UD, ... */ + uint16_t remote_lid; /* for debug */ + + /* other commands can be executed before RDMA-rd command */ + /* see the "Ordering and the Fence Indicator" section in "InfiniBand Architecture" by William T. Futral */ + uint16_t after_rdma_rd; + + uint64_t rsr_seq_num_released[(MPID_NEM_IB_COM_RDMABUF_NSEG + 63) / 64]; + +} MPID_nem_ib_com_t; + +extern int MPID_nem_ib_com_open(int ib_port, int MPID_nem_ib_com_open_flag, int *condesc); +extern int MPID_nem_ib_com_alloc(int condesc, int sz); +extern int MPID_nem_ib_com_close(int); +extern int MPID_nem_ib_com_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union ibv_gid *remote_gid); + +extern int MPID_nem_ib_com_reg_mr_connect(int condesc, void *rmem, int rkey); +extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, + int sz_hdr, void *data, int sz_data, int *copied); +extern int MPID_nem_ib_com_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, + int sz_data); +extern int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr); +//extern int MPID_nem_ib_com_isend(int condesc, uint64_t wr_id, void* hdr, int sz_hdr, void* data, int sz_data); +extern int MPID_nem_ib_com_irecv(int condesc, uint64_t wr_id); +extern int MPID_nem_ib_com_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, + uint32_t remote_qpn, uint32_t imm_data, uint64_t wr_id); +extern int MPID_nem_ib_com_udrecv(int condesc); +extern int MPID_nem_ib_com_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, + void *laddr); +extern int MPID_nem_ib_com_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, + void *laddr); +extern int MPID_nem_ib_com_poll_cq(int which_cq, struct ibv_wc *wc, int *result); + +extern int MPID_nem_ib_com_obtain_pointer(int condesc, MPID_nem_ib_com_t ** MPID_nem_ib_com); + +/* for ib_reg_mr.c */ +extern int MPID_nem_ib_com_reg_mr(void *addr, int len, struct ibv_mr **mr); +extern int MPID_nem_ib_com_dereg_mr(struct ibv_mr *mr); + +extern int MPID_nem_ib_com_get_info_conn(int condesc, int key, void *out, uint32_t out_len); +extern int MPID_nem_ib_com_get_info_mr(int condesc, int memid, int key, void *out, int out_len); + +extern int MPID_nem_ib_com_sseq_num_get(int condesc, int *seq_num); +extern int MPID_nem_ib_com_lsr_seq_num_tail_get(int condesc, int **seq_num); +extern int MPID_nem_ib_com_rsr_seq_num_tail_get(int condesc, int **seq_num); +extern int MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get(int condesc, int **seq_num); +extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(int condesc, int *notify_rate); +extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(int condesc, int **rstate); +extern int MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get(int condesc, int **lstate); + +extern int MPID_nem_ib_comMemInfo(int, int, void **, size_t *, int *); +extern char *MPID_nem_ib_com_strerror(int errno); +extern int dflag; + +extern int MPID_nem_ib_com_mem_rdmawr_from(int condesc, void **out); +extern int MPID_nem_ib_com_mem_rdmawr_to(int condesc, int seq_num, void **out); +extern int MPID_nem_ib_com_mem_udwr_from(int condesc, void **out); +extern int MPID_nem_ib_com_mem_udwr_to(int condesc, void **out); + +/* ib_reg_mr.c */ +extern void MPID_nem_ib_com_register_cache_init(void); +extern void MPID_nem_ib_com_register_cache_destroy(void); +extern struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len); + +/* dcfa_ctlmsg.c */ +extern int MPID_nem_ib_com_udbuf_init(void *q); + +#define MPID_NEM_IB_COM_RC_SHARED_RCQ 0 +#define MPID_NEM_IB_COM_RC_SHARED_SCQ 1 +#define MPID_NEM_IB_COM_UD_SHARED_RCQ 2 +#define MPID_NEM_IB_COM_UD_SHARED_SCQ 3 +#define MPID_NEM_IB_COM_RC_SHARED_SCQ_LMT_PUT 4 + +/* flag for open */ +#define MPID_NEM_IB_COM_OPEN_RC 0x01 +/* for MPI control message, eager send, rendezvous protocol, + so via RC-send/recv or RDMA-write/RDMA-read */ + +#define MPID_NEM_IB_COM_OPEN_UD 0x02 +/* obsolete, to wait for you-to-me QP to become RTR state + so via UD-send/recv */ + +#define MPID_NEM_IB_COM_OPEN_RC_LMT_PUT 0x03 +/* obsolete, tried to use different CQ for LMT-PUT protocol for speed */ + +#define MPID_NEM_IB_COM_OPEN_SCRATCH_PAD 0x04 +/* obsolete, to wait for you-to-me QP to become RTR state + so via RDMA-write */ + +#define MPID_NEM_IB_COM_ERR_SETANDJUMP(errno, stmt) { stmt; ibcom_errno = errno; goto fn_fail; } +#define MPID_NEM_IB_COM_ERR_CHKANDJUMP(cond, errno, stmt) if (cond) { stmt; ibcom_errno = errno; goto fn_fail; } + +#define MPID_NEM_IB_COM_QKEY 0x1234 +#define MPID_NEM_IB_COM_MAGIC 0x55 + +typedef struct MPID_nem_ib_sz_hdrmagic_t { + uint32_t sz; + uint32_t magic; +} MPID_nem_ib_sz_hdrmagic_t; + + +typedef struct MPID_nem_ib_tailmagic_t { + uint8_t magic; + //uint32_t traits; /* for debug */ +} MPID_nem_ib_tailmagic_t; + +#define MPID_NEM_IB_SZ_DATA_POW2(sz) \ + for(sz_data_pow2 = 15; sz_data_pow2 < (sz); sz_data_pow2 = ((((sz_data_pow2 + 1) << 1) - 1) > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_tailmagic_t)) ? MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_tailmagic_t) : (((sz_data_pow2 + 1) << 1) - 1)) { } \ + if (sz_data_pow2 > MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_tailmagic_t)) { printf("assertion failed\n"); }; \ + +#define MPID_NEM_IB_MAX_DATA_POW2 (MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_tailmagic_t)) + +typedef struct MPID_nem_ib_com_qp_state_t { + uint32_t state; +} MPID_nem_ib_com_qp_state_t; + +#define MPID_NEM_IB_COM_QP_STATE_RTR 0x12345678 +#define MPID_NEM_IB_COM_SZ_MPI_HEADER 48 +#define MPID_NEM_IB_COM_AMT_SLACK (MPID_NEM_IB_COM_RDMABUF_NSEG > 128 ? 1 : 1) diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h new file mode 100644 index 0000000..1be6a25 --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h @@ -0,0 +1,645 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2012 NEC Corporation + * Author: Masamichi Takagi + * See COPYRIGHT in top-level directory. + */ + +#ifndef IB_IMPL_H_INCLUDED +#define IB_IMPL_H_INCLUDED + +#include "mpid_nem_impl.h" +#include "ib_ibcom.h" +#include <sys/types.h> +#include <errno.h> +#include <linux/mman.h> /* make it define MAP_ANONYMOUS */ +#include <sys/mman.h> + +#define MPID_NEM_IB_LMT_GET_CQE /* detect RDMA completion by CQE */ +#define MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE +/* lmt-put: + (1) receiver sends cts to sender (2) sender RDMA-write to receiver + (3) sender fetch CQE (4) receiver polls on end-flag +*/ +//#define MPID_NEM_IB_ONDEMAND + +typedef struct { + union ibv_gid gid; + uint16_t lid; + uint32_t qpn; +} MPID_nem_ib_conn_ud_t; + +typedef struct { + int fd, fd_lmt_put; + MPIDI_VC_t *vc; +} MPID_nem_ib_conn_t; + +/* see src/mpid/ch3/channels/nemesis/include/mpid_nem_generic_queue.h */ +typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_ib_sendq_t; + +/* The vc provides a generic buffer in which network modules can store + private fields This removes all dependencies from the VC struction + on the network module, facilitating dynamic module loading. */ +typedef struct { + MPID_nem_ib_conn_t *sc; + int pending_sends; /* number of send in flight */ + MPID_nem_ib_com_t *ibcom, *ibcom_lmt_put; + MPID_nem_ib_sendq_t sendq; /* overflow queue for IB commands */ + MPID_nem_ib_sendq_t sendq_lmt_put; + int is_connected; /* dynamic connection, checked in iSendContig, protocol processed there and in progress engine */ +} MPID_nem_ib_vc_area; + +/* macro for secret area in vc */ +#define VC_CH(vc) ((MPIDI_CH3I_VC *)&(vc)->ch) +#define VC_IB(vc) ((MPID_nem_ib_vc_area *)VC_CH((vc))->netmod_area.padding) +#define VC_FIELD(vcp, field) (((MPID_nem_ib_vc_area *)VC_CH(((vcp)))->netmod_area.padding)->field) + +/* The req provides a generic buffer in which network modules can store + private fields This removes all dependencies from the req structure + on the network module, facilitating dynamic module loading. */ +typedef struct { + int seq_num; /* NOT USED, DELETE IT: sequence number of SR which RDMA-RD for lmt releases in ib_poll */ + struct MPID_Request *lmt_next; /* for lmtq */ + struct MPID_Request *sendq_next; /* for sendq */ + void *lmt_raddr; /* remember this for sendq, it might be better to use sreq->dev.iov[0].MPID_IOV_BUF instead */ + uint32_t lmt_rkey; /* remember this for sendq, survive over lrecv and referenced when dequeueing from sendq */ + uint32_t lmt_szsend; /* remember this for sendq */ + uint8_t lmt_tail, lmt_sender_tail, lmt_receiver_tail; /* survive over lrecv and referenced when polling */ + MPI_Aint lmt_dt_true_lb; /* to locate the last byte of receive buffer */ + void *lmt_write_to_buf; /* user buffer or temporary buffer for pack and remember it for lmt_orderq */ + void *lmt_pack_buf; /* to pack non-contiguous data */ +} MPID_nem_ib_req_area; + +/* macro for secret area in req */ +#define REQ_IB(req) ((MPID_nem_ib_req_area *)(&(req)->ch.netmod_area.padding)) +#define REQ_FIELD(reqp, field) (((MPID_nem_ib_req_area *)((reqp)->ch.netmod_area.padding))->field) + +/* see src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h */ +/* sreq is never enqueued into posted-queue nor unexpected-queue, so we can reuse sreq->dev.next */ +#define MPID_nem_ib_sendq_empty(q) GENERICM_Q_EMPTY (q) +#define MPID_nem_ib_sendq_head(q) GENERICM_Q_HEAD (q) +#define MPID_nem_ib_sendq_next_field(ep, next_field) REQ_FIELD(ep, next_field) +#define MPID_nem_ib_sendq_next(ep) REQ_FIELD(ep, sendq_next) +//#define MPID_nem_ib_sendq_next(ep) (ep->dev.next) /*takagi*/ +#define MPID_nem_ib_sendq_enqueue(qp, ep) GENERICM_Q_ENQUEUE (qp, ep, MPID_nem_ib_sendq_next_field, sendq_next); +#define MPID_nem_ib_sendq_enqueue_at_head(qp, ep) GENERICM_Q_ENQUEUE_AT_HEAD(qp, ep, MPID_nem_ib_sendq_next_field, sendq_next); +#define MPID_nem_ib_sendq_dequeue(qp, ep) GENERICM_Q_DEQUEUE (qp, ep, MPID_nem_ib_sendq_next_field, sendq_next); + +/* see src/mpid/ch3/channels/nemesis/include/mpid_nem_generic_queue.h */ +typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_ib_lmtq_t; + +/* connection manager */ +typedef struct { + int remote_rank; + uint32_t type; /* SYN */ + uint32_t qpn; /* QPN for eager-send channel */ + uint32_t rkey; /* key for RDMA-write-to buffer of eager-send channel */ + void *rmem; /* address of RDMA-write-to buffer of eager-send channel */ +} MPID_nem_ib_cm_pkt_syn_t; + +typedef struct { + int remote_rank; + uint32_t type; /* SYNACK */ + uint32_t qpn; /* QPN for eager-send channel */ + uint32_t rkey; /* key for RDMA-write-to buffer of eager-send channel */ + void *rmem; /* address of RDMA-write-to buffer of eager-send channel */ +} MPID_nem_ib_cm_pkt_synack_t; + +typedef union { + MPID_nem_ib_cm_pkt_syn_t syn; + MPID_nem_ib_cm_pkt_synack_t synack; +} MPID_nem_ib_cm_pkt_t; + +typedef struct MPID_nem_ib_cm_sendq_entry { + MPID_nem_ib_cm_pkt_t pending_pkt; + struct MPID_nem_ib_cm_sendq_entry *sendq_next; /* for software command queue */ +} MPID_nem_ib_cm_sendq_entry_t; + +#ifdef MPID_NEM_IB_ONDEMAND +typedef struct { + char *data; + int length; + int max_length; +} MPID_nem_ib_cm_map_t; + +typedef struct { + uint32_t type; + uint32_t qpnum; + uint16_t lid; + union ibv_gid gid; + void *rmem; + uint32_t rkey; +} MPID_nem_ib_cm_cmd_t; +#endif + +typedef GENERIC_Q_DECL(struct MPID_Request) MPID_nem_ib_cm_sendq_t; + +#define MPID_nem_ib_cm_sendq_empty(q) GENERICM_Q_EMPTY (q) +#define MPID_nem_ib_cm_sendq_head(q) GENERICM_Q_HEAD (q) +#define MPID_nem_ib_cm_sendq_next_field(ep, next_field) ((ep)->next_field) +#define MPID_nem_ib_cm_sendq_next(ep) ((ep)->sendq_next) +#define MPID_nem_ib_cm_sendq_enqueue(qp, ep) GENERICM_Q_ENQUEUE (qp, ep, MPID_nem_ib_cm_sendq_next_field, sendq_next); +#define MPID_nem_ib_cm_sendq_enqueue_at_head(qp, ep) GENERICM_Q_ENQUEUE_AT_HEAD(qp, ep, MPID_nem_ib_cm_sendq_next_field, sendq_next); +#define MPID_nem_ib_cm_sendq_dequeue(qp, ep) GENERICM_Q_DEQUEUE (qp, ep, MPID_nem_ib_cm_sendq_next_field, sendq_next); + +/* see src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h */ +/* TODO: rreq for rendezvous is dequeued from posted-queue nor unexpected-queue when do_cts is called, + so we can reuse rreq->dev.next */ +#define MPID_nem_ib_lmtq_empty(q) GENERICM_Q_EMPTY(q) +#define MPID_nem_ib_lmtq_head(q) GENERICM_Q_HEAD(q) +#define MPID_nem_ib_lmtq_next_field(ep, next_field) REQ_FIELD(ep, next_field) +#define MPID_nem_ib_lmtq_next(ep) REQ_FIELD(ep, lmt_next) +#define MPID_nem_ib_lmtq_enqueue(qp, ep) GENERICM_Q_ENQUEUE(qp, ep, MPID_nem_ib_lmtq_next_field, lmt_next); + +#define MPID_nem_ib_diff32(a, b) ((uint32_t)((a + (1ULL<<32) - b) & ((1ULL<<32)-1))) +#define MPID_nem_ib_sendq_ready_to_send_head(vc_ib) (vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY && MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY && MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG) +#define MPID_nem_ib_sendq_ready_to_send_head_lmt_put(vc_ib) (vc_ib->ibcom->ncom_lmt_put < MPID_NEM_IB_COM_MAX_SQ_CAPACITY && MPID_nem_ib_ncqe_lmt_put < MPID_NEM_IB_COM_MAX_CQ_CAPACITY) + +/* counting bloom filter to detect multiple lmt-sends in one send-wait period to + avoid overwriting the last byte in the receive buffer */ +#define MPID_nem_ib_cbf_nslot 16 /* slots */ +#define MPID_nem_ib_cbf_bitsperslot 4 /* one slot can accomodate multiple bits */ +#define MPID_nem_ib_cbf_lognslot 4 +#define MPID_nem_ib_cbf_nhash 3 /* number of hash functions */ +#define MPID_nem_ib_getpos \ + int pos_8b = pos / (8 / MPID_nem_ib_cbf_bitsperslot);\ + assert(0 <= pos_8b && pos_8b < MPID_nem_ib_cbf_nslot * MPID_nem_ib_cbf_bitsperslot / 8);\ + int pos_bps = pos & (8 / MPID_nem_ib_cbf_bitsperslot - 1); +#define MPID_nem_ib_shift \ + ((array[pos_8b] >> (pos_bps * MPID_nem_ib_cbf_bitsperslot)) & ((1ULL<<MPID_nem_ib_cbf_bitsperslot) - 1)) +#define MPID_nem_ib_maskset \ + array[pos_8b] &= ~(((1ULL<<MPID_nem_ib_cbf_bitsperslot) - 1) << (pos_bps * MPID_nem_ib_cbf_bitsperslot)); \ + array[pos_8b] |= (bits & ((1ULL<<MPID_nem_ib_cbf_bitsperslot)-1)) << (pos_bps * MPID_nem_ib_cbf_bitsperslot) +static inline int MPID_nem_ib_cbf_get(uint8_t * array, int pos) +{ + MPID_nem_ib_getpos; + return MPID_nem_ib_shift; +} + +static inline void MPID_nem_ib_cbf_set(uint8_t * array, int pos, uint16_t bits) +{ + MPID_nem_ib_getpos; + MPID_nem_ib_maskset; +} + +static inline void MPID_nem_ib_cbf_inc(uint8_t * array, int pos) +{ + MPID_nem_ib_getpos; + int16_t bits = MPID_nem_ib_shift; + assert(bits != (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1); + bits++; + MPID_nem_ib_maskset; +} + +static inline void MPID_nem_ib_cbf_dec(uint8_t * array, int pos) +{ + MPID_nem_ib_getpos; + int16_t bits = MPID_nem_ib_shift; + assert(bits != 0); + bits--; + MPID_nem_ib_maskset; +} + +static inline int MPID_nem_ib_cbf_hash1(uint64_t addr) +{ + return + (((addr >> (MPID_nem_ib_cbf_lognslot * 0)) & (MPID_nem_ib_cbf_nslot - 1)) ^ + ((addr >> (MPID_nem_ib_cbf_lognslot * 3)) & (MPID_nem_ib_cbf_nslot - 1)) ^ + ((addr >> (MPID_nem_ib_cbf_lognslot * 6)) & (MPID_nem_ib_cbf_nslot - 1)) + + 1) & (MPID_nem_ib_cbf_nslot - 1); +} + +static inline int MPID_nem_ib_cbf_hash2(uint64_t addr) +{ + /* adding one because addr tends to have a postfix of "fff" */ + return + (((addr >> (MPID_nem_ib_cbf_lognslot * 1)) & (MPID_nem_ib_cbf_nslot - 1)) ^ + ((addr >> (MPID_nem_ib_cbf_lognslot * 4)) & (MPID_nem_ib_cbf_nslot - 1)) ^ + ((addr >> (MPID_nem_ib_cbf_lognslot * 7)) & (MPID_nem_ib_cbf_nslot - 1)) + + 1) & (MPID_nem_ib_cbf_nslot - 1); +} + +static inline int MPID_nem_ib_cbf_hash3(uint64_t addr) +{ + /* adding two because addr tends to have a postfix of "fff" */ + return + (((addr >> (MPID_nem_ib_cbf_lognslot * 2)) & (MPID_nem_ib_cbf_nslot - 1)) ^ + ((addr >> (MPID_nem_ib_cbf_lognslot * 5)) & (MPID_nem_ib_cbf_nslot - 1)) ^ + ((addr >> (MPID_nem_ib_cbf_lognslot * 8)) & (MPID_nem_ib_cbf_nslot - 1)) + + 2) & (MPID_nem_ib_cbf_nslot - 1); + +} + +static inline void MPID_nem_ib_cbf_add(uint64_t addr, uint8_t * array) +{ + //dprintf("cbf_add,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_ib_cbf_hash1(addr), MPID_nem_ib_cbf_hash2(addr), MPID_nem_ib_cbf_hash3(addr)); + //dprintf("cbf_add,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); + MPID_nem_ib_cbf_inc(array, MPID_nem_ib_cbf_hash1(addr)); + MPID_nem_ib_cbf_inc(array, MPID_nem_ib_cbf_hash2(addr)); + MPID_nem_ib_cbf_inc(array, MPID_nem_ib_cbf_hash3(addr)); + //dprintf("cbf_add,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); +} + +static inline void MPID_nem_ib_cbf_delete(uint64_t addr, uint8_t * array) +{ + //dprintf("cbf_delete,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_ib_cbf_hash1(addr), MPID_nem_ib_cbf_hash2(addr), MPID_nem_ib_cbf_hash3(addr)); + //dprintf("cbf_delete,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); + MPID_nem_ib_cbf_dec(array, MPID_nem_ib_cbf_hash1(addr)); + MPID_nem_ib_cbf_dec(array, MPID_nem_ib_cbf_hash2(addr)); + MPID_nem_ib_cbf_dec(array, MPID_nem_ib_cbf_hash3(addr)); + //dprintf("cbf_delete,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); +} + +static inline int MPID_nem_ib_cbf_query(uint64_t addr, uint8_t * array) +{ + //dprintf("cbf_query,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_ib_cbf_hash1(addr), MPID_nem_ib_cbf_hash2(addr), MPID_nem_ib_cbf_hash3(addr)); + //dprintf("cbf_query,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); + return + MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)) > 0 && + MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)) > 0 && + MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr)) > 0; +} + +static inline int MPID_nem_ib_cbf_would_overflow(uint64_t addr, uint8_t * array) +{ + //dprintf("cbf_would_overflow,addr=%08lx,%08x,%08x,%08x\n", addr, MPID_nem_ib_cbf_hash1(addr), MPID_nem_ib_cbf_hash2(addr), MPID_nem_ib_cbf_hash3(addr)); + //dprintf("cbf_would_overflow,%d,%d,%d\n", MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash1(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash2(addr)), MPID_nem_ib_cbf_get(array, MPID_nem_ib_cbf_hash3(addr))); + return + MPID_nem_ib_cbf_get(array, + MPID_nem_ib_cbf_hash1(addr)) == + (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1 || + MPID_nem_ib_cbf_get(array, + MPID_nem_ib_cbf_hash2(addr)) == + (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1 || + MPID_nem_ib_cbf_get(array, + MPID_nem_ib_cbf_hash3(addr)) == + (1ULL << MPID_nem_ib_cbf_bitsperslot) - 1; +} + +/* functions */ +uint8_t MPID_nem_ib_rand(void); +uint64_t MPID_nem_ib_rdtsc(void); +int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p); +int MPID_nem_ib_finalize(void); +int MPID_nem_ib_drain_scq(int dont_call_progress); +int MPID_nem_ib_drain_scq_lmt_put(void); +int MPID_nem_ib_drain_scq_scratch_pad(void); +int MPID_nem_ib_poll(int in_blocking_poll); +int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc); + +int MPID_nem_ib_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p); +int MPID_nem_ib_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc); +int MPID_nem_ib_vc_init(MPIDI_VC_t * vc); +int MPID_nem_ib_vc_destroy(MPIDI_VC_t * vc); +int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc); +int MPID_nem_ib_pkthandler_init(MPIDI_CH3_PktHandler_Fcn * pktArray[], int arraySize); + +int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header, + MPIDI_msg_sz_t hdr_sz); + +/* CH3 send/recv functions */ +int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, + MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz); +int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, + MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr); + +/* used by ib_poll.c */ +int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib); + +/* CH3--lmt send/recv functions */ +int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, + struct MPID_Request *req); +int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, + void *write_to_buf); +int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie); +int MPID_nem_ib_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie); +int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req); +int MPID_nem_ib_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req); +int MPID_nem_ib_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *req); +int MPID_nem_ib_lmt_vc_terminated(struct MPIDI_VC *vc); +/* overriding functions + initialize the value of a member named "recv_posted" + in BSS-variable named "comm_ops" with type of MPIDI_Comm_ops_t + to "MPID_nem_ib_recv_posted" in ib_init.c + MPIDI_Comm_ops_t is defined in src/mpid/ch3/include/mpidimpl.h */ +int MPID_nem_ib_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req); +int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data); + +void pht_update(uint64_t vpc, uint32_t hist, int32_t taken); +int pht_pred(uint64_t vpc, uint32_t hist); + +/* Keys for business cards */ +#define MPID_NEM_IB_GID_KEY "gid" +#define MPID_NEM_IB_LID_KEY "lid" +#define MPID_NEM_IB_QPN_KEY "qpn" +#define MPID_NEM_IB_RKEY_KEY "rkey" +#define MPID_NEM_IB_RMEM_KEY "rmem" + +#define MPID_NEM_IB_RECV_MAX_PKT_LEN 1024 + +extern int MPID_nem_ib_conn_ud_fd; +extern MPID_nem_ib_com_t *MPID_nem_ib_conn_ud_ibcom; +extern MPID_nem_ib_conn_ud_t *MPID_nem_ib_conn_ud; +extern MPID_nem_ib_conn_t *MPID_nem_ib_conns; +extern MPIDI_VC_t **MPID_nem_ib_pollingset; +extern int *MPID_nem_ib_scratch_pad_fds; +extern int MPID_nem_ib_npollingset; +extern void *MPID_nem_ib_fl[18]; +extern int MPID_nem_ib_nranks; +//extern char *MPID_nem_ib_recv_buf; +extern int MPID_nem_ib_myrank; +extern uint64_t MPID_nem_ib_tsc_poll; /* to throttle ib_poll in recv_posted (in ib_poll.c) */ +extern int MPID_nem_ib_ncqe; /* for lazy poll scq */ +extern int MPID_nem_ib_ncqe_lmt_put; /* lmt-put uses another QP, SQ, CQ to speed-up fetching CQE */ +#ifdef MPID_NEM_IB_ONDEMAND +extern MPID_nem_ib_cm_map_t MPID_nem_ib_cm_state; +extern int MPID_nem_ib_ncqe_connect; /* couting outstanding connection requests */ +#endif +extern int MPID_nem_ib_ncqe_scratch_pad; +extern int MPID_nem_ib_ncqe_to_drain; /* count put in lmt-put-done protocol */ +extern int MPID_nem_ib_ncqe_nces; /* counting non-copied eager-send */ +extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq; /* poll queue for lmt */ +extern MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq; /* force order when two or more rts_to_sender randomizes the last byte of receive buffer */ +extern MPID_nem_ib_vc_area *MPID_nem_ib_debug_current_vc_ib; + +/* to detect multiple lmt-sends in one send-wait period to + avoid overwriting the last byte in the receive buffer */ +extern uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * + MPID_nem_ib_cbf_bitsperslot / 8]; + +#define MPID_NEM_IB_MAX_POLLINGSET 65536 + +/* xfer.c manages memory region using memid */ +#define MPID_NEM_IB_MEMID_RDMA 0 + +/* command using IB UD */ +#define MPID_NEM_IB_SYNC_SYN 0 +#define MPID_NEM_IB_SYNC_SYNACK 1 +#define MPID_NEM_IB_SYNC_NACK 2 + +#define MPID_NEM_IB_EAGER_MAX_MSG_SZ (MPID_NEM_IB_COM_RDMABUF_SZSEG/*1024*/-sizeof(MPIDI_CH3_Pkt_t)+sizeof(MPIDI_CH3_Pkt_eager_send_t)-sizeof(MPID_nem_ib_sz_hdrmagic_t)-sizeof(MPID_nem_ib_pkt_prefix_t)-sizeof(MPID_nem_ib_tailmagic_t)) /* when > this size, lmt is used. see src/mpid/ch3/src/mpid_isend.c */ +#define MPID_NEM_IB_POLL_PERIOD_RECV_POSTED 2000 /* minimum period from previous ib_poll to ib_poll in recv_posted */ +#define MPID_NEM_IB_POLL_PERIOD_SEND_POSTED 2000 + +typedef struct { + void *addr; + uint32_t rkey; + int seq_num_tail; /* notify RDMA-write-to buffer occupation */ + uint8_t tail; /* last word of payload */ +} MPID_nem_ib_lmt_cookie_t; + +typedef enum MPID_nem_ib_pkt_subtype { + MPIDI_NEM_IB_PKT_EAGER_SEND, + MPIDI_NEM_IB_PKT_PUT, + MPIDI_NEM_IB_PKT_ACCUMULATE, + MPIDI_NEM_IB_PKT_GET, + MPIDI_NEM_IB_PKT_GET_RESP, + MPIDI_NEM_IB_PKT_LMT_GET_DONE, + MPIDI_NEM_IB_PKT_REQ_SEQ_NUM, + MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM, + MPIDI_NEM_IB_PKT_CHG_RDMABUF_OCC_NOTIFY_STATE, + MPIDI_NEM_IB_PKT_NUM_PKT_HANDLERS +} MPID_nem_ib_pkt_subtype_t; + +/* derived from MPID_nem_pkt_netmod_t */ +typedef struct MPID_nem_ib_pkt_prefix { + MPID_nem_pkt_type_t type; + unsigned subtype; + /* additional field */ + int seq_num_tail; +} MPID_nem_ib_pkt_prefix_t; + +/* derived from MPID_nem_pkt_netmod_t and MPID_nem_pkt_lmt_done_t */ +typedef struct MPID_nem_ib_pkt_lmt_get_done { + MPID_nem_pkt_type_t type; + unsigned subtype; + /* additional field */ + MPI_Request req_id; + int seq_num_tail; +} MPID_nem_ib_pkt_lmt_get_done_t; + +/* derived from MPID_nem_pkt_netmod_t */ +typedef struct MPID_nem_ib_pkt_req_seq_num_t { + MPID_nem_pkt_type_t type; + unsigned subtype; + /* additional field */ + int seq_num_tail; +} MPID_nem_ib_pkt_req_seq_num_t; + +/* derived from MPID_nem_pkt_netmod_t */ +typedef struct MPID_nem_ib_pkt_reply_seq_num_t { + MPID_nem_pkt_type_t type; + unsigned subtype; + /* additional field */ + int seq_num_tail; +} MPID_nem_ib_pkt_reply_seq_num_t; + +/* derived from MPID_nem_pkt_netmod_t */ +typedef struct MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t { + MPID_nem_pkt_type_t type; + unsigned subtype; + /* additional field */ + int state; +} MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t; + +int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); +int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); +int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); +int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); +int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen /* out */ , + MPID_Request ** rreqp /* out */); +int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); +int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); +int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); +int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp); +int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc, + MPIDI_CH3_Pkt_t * pkt, + MPIDI_msg_sz_t * buflen, + MPID_Request ** rreqp); + +/* MPID_nem_ib_PktHandler_lmt_done is a wrapper of pkt_DONE_handler and calls it */ +/* pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) is not exported */ +int pkt_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, + MPID_Request ** rreqp); + + +#define MPID_nem_ib_send_req_seq_num(vc) do { \ + MPID_PKT_DECL_CAST(_upkt, MPID_nem_ib_pkt_req_seq_num_t, _pkt); \ + MPID_Request *_req; \ + \ + MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending req_seq_num packet"); \ + MPIDI_Pkt_init(_pkt, MPIDI_NEM_PKT_NETMOD); \ + _pkt->subtype = MPIDI_NEM_IB_PKT_REQ_SEQ_NUM; \ + \ + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); \ + _pkt->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; \ + vc_ib->ibcom->rsr_seq_num_tail_last_sent = vc_ib->ibcom->rsr_seq_num_tail; \ + \ + mpi_errno = MPIDI_CH3_iStartMsg((vc), _pkt, sizeof(*_pkt), &_req); \ + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_send_req_seq_num"); \ + if (_req != NULL) { \ + MPIU_ERR_CHKANDJUMP(_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_send_req_seq_num"); \ + MPID_Request_release(_req); \ + } \ + } while (0) + +#define MPID_nem_ib_send_reply_seq_num(vc) do { \ + MPID_PKT_DECL_CAST(_upkt, MPID_nem_ib_pkt_reply_seq_num_t, _pkt); \ + MPID_Request *_req; \ + \ + MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending reply_seq_num packet"); \ + MPIDI_Pkt_init(_pkt, MPIDI_NEM_PKT_NETMOD); \ + _pkt->subtype = MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM; \ + \ + int *rsr_seq_num_tail; \ + ibcom_errno = MPID_nem_ib_com_rsr_seq_num_tail_get(VC_FIELD(vc, sc->fd), &rsr_seq_num_tail); \ + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rsr_seq_num_tail_get"); \ + _pkt->seq_num_tail = *rsr_seq_num_tail; \ + \ + int *rsr_seq_num_tail_last_sent; \ + ibcom_errno = MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get(VC_FIELD(vc, sc->fd), &rsr_seq_num_tail_last_sent); \ + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get"); \ + *rsr_seq_num_tail_last_sent = *rsr_seq_num_tail; \ +\ + mpi_errno = MPIDI_CH3_iStartMsg((vc), _pkt, sizeof(*_pkt), &_req); \ + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_send_reply_seq_num"); \ + if (_req != NULL) { \ + MPIU_ERR_CHKANDJUMP(_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_send_reply_seq_num"); \ + MPID_Request_release(_req); \ + } \ + } while (0) + +#define MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc, _state) do { \ + MPID_PKT_DECL_CAST(_upkt, MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t, _pkt); \ + MPID_Request *_req; \ + \ + MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending change_rdmabuf_occupancy_notify_state packet"); \ + MPIDI_Pkt_init(_pkt, MPIDI_NEM_PKT_NETMOD); \ + _pkt->subtype = MPIDI_NEM_IB_PKT_CHG_RDMABUF_OCC_NOTIFY_STATE; \ + _pkt->state = _state; \ + \ + mpi_errno = MPIDI_CH3_iStartMsg((vc), _pkt, sizeof(*_pkt), &_req); \ + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state"); \ + if (_req != NULL) { \ + MPIU_ERR_CHKANDJUMP(_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state"); \ + MPID_Request_release(_req); \ + } \ + } while (0) + +#define MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, lsr_seq_num_tail) \ + do { \ + int lsr_seq_num_head; \ + /* sequence number of (largest) in-flight send command */ \ + ibcom_errno = MPID_nem_ib_com_sseq_num_get(vc_ib->sc->fd, &lsr_seq_num_head); \ + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_sseq_num_get"); \ + \ + int *rdmabuf_occupancy_notify_rstate; \ + ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &rdmabuf_occupancy_notify_rstate); \ + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get"); \ + \ + /*dprintf("notify_policy_lw,head=%d,tail=%d,lw=%d\n", lsr_seq_num_head, *lsr_seq_num_tail, MPID_NEM_IB_COM_RDMABUF_LOW_WATER_MARK);*/ \ + /* if the number of occupied slot of RDMA-write-to buffer have got below the low water-mark */ \ + if (*rdmabuf_occupancy_notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW && \ + MPID_nem_ib_diff32(lsr_seq_num_head, *lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_LOW_WATER_MARK) { \ + dprintf("changing notify_rstate\n"); \ + /* remember remote notifying policy so that local can know when to change remote policy back to HW */ \ + *rdmabuf_occupancy_notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW; \ + /* change remote notifying policy of RDMA-write-to buf occupancy info */ \ + MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc, MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW); \ + } \ + } while (0) + +#define MPID_nem_ib_lmt_send_GET_DONE(vc, rreq) do { \ + MPID_PKT_DECL_CAST(_upkt, MPID_nem_ib_pkt_lmt_get_done_t, _done_pkt); \ + MPID_Request *_done_req; \ + \ + MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending rndv DONE packet"); \ + MPIDI_Pkt_init(_done_pkt, MPIDI_NEM_PKT_NETMOD); \ + _done_pkt->subtype = MPIDI_NEM_IB_PKT_LMT_GET_DONE;\ + _done_pkt->req_id = (rreq)->ch.lmt_req_id; \ + /* embed SR occupancy information */ \ + _done_pkt->seq_num_tail = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ + \ + /* remember the last one sent */ \ + VC_FIELD(vc, ibcom->rsr_seq_num_tail_last_sent) = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ + \ + mpi_errno = MPIDI_CH3_iStartMsg((vc), _done_pkt, sizeof(*_done_pkt), &_done_req); \ + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_lmt_send_GET_DONE"); \ + if (_done_req != NULL) \ + { \ + MPIU_ERR_CHKANDJUMP(_done_req->status.MPI_ERROR, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_lmt_send_GET_DONE"); \ + MPID_Request_release(_done_req); \ + } \ + } while (0) + +#define MPID_NEM_IB_MAX(a, b) ((a) > (b) ? (a) : (b)) + +static inline void *MPID_nem_ib_stmalloc(size_t _sz) +{ + size_t sz = _sz; + int i = 0; + do { + i++; + sz >>= 1; + } while (sz > 0); + if (i < 12) { + return malloc(sz); + } + if (i > 30) { + return mmap(0, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + } + int ndx = i - 12; + void *slot; + if (MPID_nem_ib_fl[ndx]) { + slot = MPID_nem_ib_fl[ndx]; + if (MPID_nem_ib_myrank == 1) { + //printf("stmalloc,reuse %p,%08x\n", slot, (int)_sz); + } + MPID_nem_ib_fl[ndx] = *((void **) MPID_nem_ib_fl[ndx]); + } + else { + slot = mmap(0, 1 << i, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (MPID_nem_ib_myrank == 1) { + //printf("stmalloc,new %p,%08x\n", slot, (int)_sz); + } + } + return slot; +} + +static inline void MPID_nem_ib_stfree(void *ptr, size_t sz) +{ + if (MPID_nem_ib_myrank == 1) { + //printf("stfree,%p,%08x\n", ptr, (int)sz); + } + int i = 0; + do { + i++; + sz >>= 1; + } while (sz > 0); + if (i < 12) { + free(ptr); + goto fn_exit; + } + if (i > 30) { + munmap(ptr, sz); + goto fn_exit; + } + int ndx = i - 12; + *((void **) ptr) = MPID_nem_ib_fl[ndx]; + MPID_nem_ib_fl[ndx] = ptr; + fn_exit:; +} +#endif /* IB_IMPL_H_INCLUDED */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c new file mode 100644 index 0000000..5b96f5b --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c @@ -0,0 +1,1112 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2006 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + * + * (C) 2012 NEC Corporation + * Author: Masamichi Takagi + */ + +#include "ib_impl.h" +#ifdef USE_PMI2_API +#include "pmi2.h" +#else +#include "pmi.h" +#endif + +//#define MPID_NEM_IB_DEBUG_INIT +#ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ +#undef dprintf +#endif +#ifdef MPID_NEM_IB_DEBUG_INIT +#define dprintf printf +#else +#define dprintf(...) +#endif + +MPID_nem_netmod_funcs_t MPIDI_nem_ib_funcs = { + MPID_nem_ib_init, + MPID_nem_ib_finalize, + MPID_nem_ib_poll, + MPID_nem_ib_get_business_card, + MPID_nem_ib_connect_to_root, + MPID_nem_ib_vc_init, + MPID_nem_ib_vc_destroy, + MPID_nem_ib_vc_terminate, + NULL, /*MPID_nem_ib_anysource_iprobe */ + NULL, /*MPID_nem_ib_anysource_improbe */ +}; + +MPIDI_CH3_PktHandler_Fcn *MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_NUM_PKT_HANDLERS]; + +static MPIDI_Comm_ops_t comm_ops = { + /*NULL, */ MPID_nem_ib_recv_posted, + /* recv_posted */ + + NULL, /* send */ + NULL, /* rsend */ + NULL, /* ssend */ + NULL, /* isend */ + NULL, /* irsend */ + NULL, /* issend */ + + NULL, /* send_init */ + NULL, /* bsend_init */ + NULL, /* rsend_init */ + NULL, /* ssend_init */ + NULL, /* startall */ + + NULL, /* cancel_send */ + NULL, /* cancel_recv */ + + NULL, /* probe */ + NULL, /* iprobe */ + NULL, /* improbe */ +}; + +void *MPID_nem_ib_fl[18]; +int MPID_nem_ib_nranks; +MPID_nem_ib_conn_ud_t *MPID_nem_ib_conn_ud; +MPID_nem_ib_conn_t *MPID_nem_ib_conns; +MPIDI_VC_t **MPID_nem_ib_pollingset; +int MPID_nem_ib_conn_ud_fd; +MPID_nem_ib_com_t *MPID_nem_ib_conn_ud_MPID_nem_ib_com; +int MPID_nem_ib_npollingset; +int *MPID_nem_ib_scratch_pad_fds; +//char *MPID_nem_ib_recv_buf; +int MPID_nem_ib_myrank; +uint64_t MPID_nem_ib_tsc_poll; +int MPID_nem_ib_ncqe; +int MPID_nem_ib_ncqe_lmt_put; +#ifdef MPID_NEM_IB_ONDEMAND +MPID_nem_ib_cm_map_t MPID_nem_ib_cm_state; +int MPID_nem_ib_ncqe_connect; +#endif +int MPID_nem_ib_ncqe_scratch_pad; +int MPID_nem_ib_ncqe_to_drain; +int MPID_nem_ib_ncqe_nces; +MPID_nem_ib_lmtq_t MPID_nem_ib_lmtq = { NULL, NULL }; +MPID_nem_ib_lmtq_t MPID_nem_ib_lmt_orderq = { NULL, NULL }; +uint8_t MPID_nem_ib_lmt_tail_addr_cbf[MPID_nem_ib_cbf_nslot * MPID_nem_ib_cbf_bitsperslot / + 8] = { 0 }; +static uint32_t MPID_nem_ib_rand_next = 1; +MPID_nem_ib_vc_area *MPID_nem_ib_debug_current_vc_ib; +static int listen_fd; +static int listen_port; + +uint8_t MPID_nem_ib_rand() +{ + //return 0xaa; + MPID_nem_ib_rand_next = MPID_nem_ib_rand_next * 1103515245 + 12345; + return (MPID_nem_ib_rand_next / 65536) % 256; +} + +uint64_t MPID_nem_ib_rdtsc() +{ + uint64_t x; + __asm__ __volatile__("rdtsc; shl $32, %%rdx; or %%rdx, %%rax":"=a"(x)::"%rdx", "memory"); /* rdtsc cannot be executed earlier than here */ + return x; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_kvs_put_binary +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int MPID_nem_ib_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, int length) +{ + int mpi_errno = MPI_SUCCESS; + int pmi_errno; + char *kvs_name; + char key[256], val[256], str[256]; + int j; + + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_KVS_PUT_BINARY); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_KVS_PUT_BINARY); + + mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPIDI_PG_GetConnKVSname"); + dprintf("kvs_put_binary,kvs_name=%s\n", kvs_name); + + sprintf(key, "bc/%d/%s", from, postfix); + val[0] = 0; + for (j = 0; j < length; j++) { + sprintf(str, "%02x", buf[j]); + strcat(val, str); + } + dprintf("kvs_put_binary,rank=%d,from=%d,PMI_KVS_Put(%s, %s, %s)\n", MPID_nem_ib_myrank, from, + kvs_name, key, val); + pmi_errno = PMI_KVS_Put(kvs_name, key, val); + MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMI_KVS_Put"); + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_KVS_PUT_BINARY); + return mpi_errno; + fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_kvs_get_binary +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int MPID_nem_ib_kvs_get_binary(int from, const char *postfix, char *buf, int length) +{ + int mpi_errno = MPI_SUCCESS; + int pmi_errno; + char *kvs_name; + char key[256], val[256], str[256]; + int j; + + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_KVS_GET_BINARY); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_KVS_GET_BINARY); + + mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); + dprintf("kvs_get_binary,kvs_name=%s\n", kvs_name); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPIDI_PG_GetConnKVSname"); + + sprintf(key, "bc/%d/%s", from, postfix); + pmi_errno = PMI_KVS_Get(kvs_name, key, val, 256); + dprintf("kvs_put_binary,rank=%d,from=%d,PMI_KVS_Get(%s, %s, %s)\n", MPID_nem_ib_myrank, from, + kvs_name, key, val); + MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMS_KVS_Get"); + + dprintf("rank=%d,obtained val=%s\n", MPID_nem_ib_myrank, val); + char *strp = val; + for (j = 0; j < length; j++) { + memcpy(str, strp, 2); + str[2] = 0; + buf[j] = strtol(str, NULL, 16); + strp += 2; + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_KVS_GET_BINARY); + return mpi_errno; + fn_fail: + goto fn_exit; +} + +static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p); + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_init +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p) +{ + int mpi_errno = MPI_SUCCESS; + int ibcom_errno = 0, pmi_errno; + int ret; + int i, j, k; + int ib_port = 1; + + MPIU_CHKPMEM_DECL(7); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_INIT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_INIT); + + /* first make sure that our private fields in the vc fit into the area provided */ + MPIU_Assert(sizeof(MPID_nem_ib_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN); + + MPID_nem_ib_nranks = pg_p->size; + MPID_nem_ib_myrank = pg_rank; + MPID_nem_ib_tsc_poll = MPID_nem_ib_rdtsc(); + MPID_nem_ib_ncqe = 0; + MPID_nem_ib_ncqe_lmt_put = 0; +#ifdef MPID_NEM_IB_ONDEMAND + MPID_nem_ib_ncqe_connect = 0; +#endif + MPID_nem_ib_ncqe_scratch_pad = 0; + MPID_nem_ib_ncqe_to_drain = 0; + MPID_nem_ib_ncqe_nces = 0; + MPID_nem_ib_npollingset = 0; + +#ifdef MPID_NEM_IB_ONDEMAND + /* prepare UD QPN for dynamic connection */ + ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_UD, &MPID_nem_ib_conn_ud_fd); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); + ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conn_ud_fd, &MPID_nem_ib_conn_ud_MPID_nem_ib_com); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); + ibcom_errno = MPID_nem_ib_com_rts(MPID_nem_ib_conn_ud_fd, 0, 0, 0); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); + + for (i = 0; i < MPID_NEM_IB_COM_MAX_RQ_CAPACITY; i++) { + ibcom_errno = MPID_nem_ib_com_udrecv(MPID_nem_ib_conn_ud_fd); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_udrecv"); + } + + /* obtain gid, lid, qpn using KVS */ + MPIU_CHKPMEM_MALLOC(MPID_nem_ib_conn_ud, MPID_nem_ib_conn_ud_t *, + MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_ud_t), mpi_errno, + "ud connection table"); + memset(MPID_nem_ib_conn_ud, 0, MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_ud_t)); + + /* put bc/<my rank>/dcs/gid:lid:qpn */ + uint32_t my_qpnum; + uint16_t my_lid; + union ibv_gid my_gid; + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, + sizeof(uint32_t)); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid, + sizeof(uint16_t)); + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conn_ud_fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid, + sizeof(union ibv_gid)); + + char *kvs_name; + mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name); + char *key_dcs, val[2 * sizeof(union ibv_gid) + 1 + 4 + 1 + 8 + 1], str[9]; + + /* count maximum length of the string representation of remote_rank */ + for (i = 0, nranks = MPID_nem_ib_nranks; nranks > 0; nranks /= 10, i++) { + } + MPIU_CHKPMEM_MALLOC(key_dcs, char *, strlen("bc/") + i + strlen("/dcs/gid_lid_qpn") + 1, + mpi_errno, "connection table"); + + sprintf(key, "bc/%d/dcs/gid_lid_qpn", MPID_nem_ib_myrank); + val[0] = 0; + for (j = 0; j < sizeof(union ibv_gid); j++) { + sprintf(str, "%02x", my_gid.raw[j]); + strcat(val, str); + } + sprintf(str, ":"); + strcat(val, str); + sprintf(str, "%04x:", my_lid); + strcat(val, str); + sprintf(str, "%08x", my_qpnum); + strcat(val, str); + dprintf("rank=%d,PMI_KVS_Put(%s, %s, %s)\n", MPID_nem_ib_myrank, kvs_name, key_dcs, val); + pmi_errno = PMI_KVS_Put(kvs_name, key_dcs, val); + MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMI_KVS_Put"); + + /* wait for key-value to propagate among all ranks */ + pmi_errno = PMI_Barrier(); + MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); + + /* obtain GID, LID, QP number for remote UD QP for dynamic connection */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + if (i != MPID_nem_ib_myrank) { + sprintf(key_dcs, "bc/%d/dcs/gid_lid_qpn", i); + pmi_errno = PMI_KVS_Get(kvs_name, key_dcs, val, 256); + dprintf("pmi_errno=%d\n", pmi_errno); + MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**PMI_KVS_Get"); + dprintf("rank=%d,obtained val=%s\n", MPID_nem_ib_myrank, val); + char *strp = val; + for (j = 0; j < sizeof(union ibv_gid); j++) { + memcpy(str, strp, 2); + str[2] = 0; + MPID_nem_ib_conn_ud[i].gid.raw[j] = strtol(str, NULL, 16); + strp += 2; + } + sscanf(strp, ":%04x:%08x", &MPID_nem_ib_conn_ud[i].lid, + &MPID_nem_ib_conn_ud[i].qpn); + + dprintf("remote rank=%d,gid=", i); + for (j = 0; j < sizeof(union ibv_gid); j++) { + dprintf("%02x", MPID_nem_ib_conn_ud[i].gid.raw[j]); + } + dprintf(",lid=%04x,qpn=%08x\n", MPID_nem_ib_conn_ud[i].lid, + MPID_nem_ib_conn_ud[i].qpn); + } + } +#endif + + /* malloc scratch-pad fd */ + MPIU_CHKPMEM_MALLOC(MPID_nem_ib_scratch_pad_fds, int *, MPID_nem_ib_nranks * sizeof(int), + mpi_errno, "connection table"); + memset(MPID_nem_ib_scratch_pad_fds, 0, MPID_nem_ib_nranks * sizeof(int)); + + /* prepare scrath-pad QP and malloc scratch-pad */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_SCRATCH_PAD, &MPID_nem_ib_scratch_pad_fds[i]); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); + + ibcom_errno = + MPID_nem_ib_com_alloc(MPID_nem_ib_scratch_pad_fds[i], + MPID_nem_ib_nranks * sizeof(MPID_nem_ib_com_qp_state_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_alloc"); + } + + /* put bc/me/sp/{gid,lid} put bc/me/sp/{qpn,rmem,rkey}/you */ + int nranks; + +#ifndef MPID_NEM_IB_ONDEMAND + uint32_t my_qpnum; + uint16_t my_lid; + union ibv_gid my_gid; +#endif + void *my_rmem; + int my_rkey; + + int remote_qpnum; + uint16_t remote_lid; + union ibv_gid remote_gid; + void *remote_rmem; + int remote_rkey; + + char *remote_rank_str; + char *key_str; + + /* count maximum length of the string representation of remote_rank */ + for (i = 0, nranks = MPID_nem_ib_nranks; nranks > 0; nranks /= 10, i++) { + } + MPIU_CHKPMEM_MALLOC(remote_rank_str, char *, 1 + i + 1, mpi_errno, "connection table"); + MPIU_CHKPMEM_MALLOC(key_str, char *, strlen("sp/qpn") + 1 + i + 1, mpi_errno, + "connection table"); + + for (i = 0; i < MPID_nem_ib_nranks; i++) { + + if (i == 0) { + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_PORT_LID, + &my_lid, sizeof(uint16_t)); + dprintf("ib_init,scratch pad,lid=%04x\n", my_lid); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, "sp/lid", (uint8_t *) & my_lid, + sizeof(uint16_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_kvs_put_binary"); + + { + dprintf("ib_init,scratch pad,put <%d/sp/lid/,%04x>\n", MPID_nem_ib_myrank, + (int) my_lid); + } + + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_PORT_GID, + &my_gid, sizeof(union ibv_gid)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, "sp/gid", (uint8_t *) & my_gid, + sizeof(union ibv_gid)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_kvs_put_binary"); + + dprintf("ib_init,scratch pad,gid "); + for (k = 0; k < 16; k++) { + dprintf("%02x", (int) my_gid.raw[k]); + } + dprintf("\n"); + } + + /* put bc/me/sp/qpn/you */ + strcpy(key_str, "sp/qpn"); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + dprintf("ib_init,scratch pad,qpn=%08x\n", my_qpnum); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); + dprintf("ib_init,scratch pad,kvs put done\n"); + + strcpy(key_str, "sp/rmem"); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + + ibcom_errno = + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &my_rmem, sizeof(void *)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); + + dprintf("ib_init,scratch_pad,rmem=%p\n", my_rmem); + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rmem, + sizeof(void *)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); + + strcpy(key_str, "sp/rkey"); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + + ibcom_errno = + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_scratch_pad_fds[i], MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &my_rkey, sizeof(int)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); + dprintf("ib_init,scratch_pad,rkey=%08x\n", my_rkey); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rkey, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); + } + + /* wait until key-value propagates among all ranks */ + pmi_errno = PMI_Barrier(); + MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); + dprintf("ib_init,put KVS;barrier;\n"); + + /* make me-to-you scratch-pad QP RTS */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + if (i != MPID_nem_ib_myrank) { + + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, "sp/gid", (char *) &remote_gid, + sizeof(union ibv_gid)); + dprintf("ib_init,after kvs get\n"); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, "sp/lid", (char *) &remote_lid, sizeof(uint16_t)); + dprintf("ib_init,after kvs get\n"); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + strcpy(key_str, "sp/qpn"); + strcat(key_str, ""); /* "" or "lmt-put" */ + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_qpnum, sizeof(uint32_t)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + dprintf("ib_init,get KVS,remote_qpnum=%08x\n", remote_qpnum); + + ibcom_errno = + MPID_nem_ib_com_rts(MPID_nem_ib_scratch_pad_fds[i], remote_qpnum, remote_lid, &remote_gid); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); + + strcpy(key_str, "sp/rmem"); + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_rmem, sizeof(void *)); + dprintf("ib_init,after kvs get\n"); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + dprintf("ib_init,get KVS,remote_rmem=%p\n", remote_rmem); + + strcpy(key_str, "sp/rkey"); + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_rkey, sizeof(uint32_t)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + dprintf("ib_init,get KVS,remote_rkey=%08x\n", remote_rkey); + + ibcom_errno = + MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_scratch_pad_fds[i], remote_rmem, remote_rkey); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect"); + } + } + + /* wait until you-to-me scratch-pad QP becomes RTR */ + pmi_errno = PMI_Barrier(); + MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); + + MPIU_CHKPMEM_MALLOC(MPID_nem_ib_conns, MPID_nem_ib_conn_t *, + MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t), mpi_errno, "connection table"); + memset(MPID_nem_ib_conns, 0, MPID_nem_ib_nranks * sizeof(MPID_nem_ib_conn_t)); + + MPIU_CHKPMEM_MALLOC(MPID_nem_ib_pollingset, MPIDI_VC_t **, + MPID_NEM_IB_MAX_POLLINGSET * sizeof(MPIDI_VC_t *), mpi_errno, + "connection table"); + memset(MPID_nem_ib_pollingset, 0, MPID_NEM_IB_MAX_POLLINGSET * sizeof(MPIDI_VC_t *)); + + /* prepare eager-send QP */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[i].fd); + dprintf("init,fd=%d\n", MPID_nem_ib_conns[i].fd); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); + } + +#if 0 + for (i = 0; i < MPID_nem_ib_nranks; i++) { + ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC_LMT_PUT, &MPID_nem_ib_conns[i].fd_lmt_put); + dprintf("init,fd_lmt_put=%d\n", MPID_nem_ib_conns[i].fd_lmt_put); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); + } +#endif + + /* put bc/me/{gid,lid}, put bc/me/{qpn,rmem,rkey}/you */ + mpi_errno = MPID_nem_ib_announce_network_addr(pg_rank, bc_val_p, val_max_sz_p); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + /* wait until key-value propagates among all ranks */ + pmi_errno = PMI_Barrier(); + MPIU_ERR_CHKANDJUMP(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**PMI_Barrier"); + + /* make me-to-you eager-send QP RTS */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + if (i != MPID_nem_ib_myrank) { + + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, MPID_NEM_IB_LID_KEY, (char *) &remote_lid, + sizeof(uint16_t)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, MPID_NEM_IB_GID_KEY, (char *) &remote_gid, + sizeof(union ibv_gid)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + strcpy(key_str, MPID_NEM_IB_RMEM_KEY); + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_rmem, sizeof(void *)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + strcpy(key_str, MPID_NEM_IB_RKEY_KEY); + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_rkey, sizeof(uint32_t)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + strcpy(key_str, MPID_NEM_IB_QPN_KEY); + strcat(key_str, ""); /* "" or "lmt-put" */ + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_qpnum, sizeof(uint32_t)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + dprintf("remote_qpnum obtained=%08x\n", remote_qpnum); + + ibcom_errno = + MPID_nem_ib_com_rts(MPID_nem_ib_conns[i].fd, remote_qpnum, remote_lid, &remote_gid); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); + + /* report me-to-you eager-send QP becomes RTR */ + MPID_nem_ib_com_t *MPID_nem_ib_com_scratch_pad; + ibcom_errno = + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[i], &MPID_nem_ib_com_scratch_pad); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); + + MPID_nem_ib_com_qp_state_t state = {.state = MPID_NEM_IB_COM_QP_STATE_RTR }; + ibcom_errno = + MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[i], + (uint64_t) MPID_nem_ib_com_scratch_pad, + sizeof(MPID_nem_ib_com_qp_state_t) * MPID_nem_ib_myrank, + sizeof(MPID_nem_ib_com_qp_state_t), (void *) &state); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_put_scratch_pad"); + MPID_nem_ib_ncqe_scratch_pad += 1; + + ibcom_errno = MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[i].fd, remote_rmem, remote_rkey); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect"); + dprintf("ib_init,after mr_connect for me-to-you eager-send QP\n"); + +#if 0 + /* CQ, SQ, SCQ for lmt-put */ + strcpy(key_str, MPID_NEM_IB_QPN_KEY); + strcat(key_str, "lmt-put"); /* "" or "lmt-put" */ + sprintf(remote_rank_str, "/%x", MPID_nem_ib_myrank); + strcat(key_str, remote_rank_str); + mpi_errno = + MPID_nem_ib_kvs_get_binary(i, key_str, (char *) &remote_qpnum, sizeof(uint32_t)); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + ibcom_errno = + MPID_nem_ib_com_rts(MPID_nem_ib_conns[i].fd_lmt_put, remote_qpnum, remote_lid, &remote_gid); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); +#endif + } + } + +#if 0 /* debug */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + dprintf("init,fd[%d]=%d\n", i, MPID_nem_ib_conns[i].fd); + } +#endif + + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_INIT); + return mpi_errno; + fn_fail: + MPIU_CHKPMEM_REAP(); + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_get_business_card +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPID_nem_ib_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p) +{ + int mpi_errno = MPI_SUCCESS; + int str_errno = MPIU_STR_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_GET_BUSINESS_CARD); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_GET_BUSINESS_CARD); + dprintf("MPID_nem_ib_get_business_card,enter\n"); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_GET_BUSINESS_CARD); + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_announce_network_addr +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +static int MPID_nem_ib_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p) +{ + int mpi_errno = MPI_SUCCESS; + int str_errno = MPIU_STR_SUCCESS; + int ibcom_errno; + int i, j, nranks; + + uint32_t my_qpnum; + uint16_t my_lid; + union ibv_gid my_gid; + void *my_rmem; + int my_rkey; + char *remote_rank_str; /* perl -e '$key_str .= $remote_rank;' */ + char *key_str; + + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ANNOUNCE_NETWORK_ADDR); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_ANNOUNCE_NETWORK_ADDR); + MPIU_CHKLMEM_DECL(2); /* argument is the number of alloca */ + + /* count maximum length of the string representation of remote_rank */ + for (i = 0, nranks = MPID_nem_ib_nranks; nranks > 0; nranks /= 10, i++) { + } + MPIU_CHKLMEM_MALLOC(remote_rank_str, char *, i + 1, mpi_errno, "key_str"); /* alloca */ + MPIU_CHKLMEM_MALLOC(key_str, char *, strlen(MPID_NEM_IB_QPN_KEY) + i + 1, mpi_errno, "key_str"); /* alloca */ + + /* We have one local qp and remote qp for each rank-pair, + * so a rank should perform + * remote_qpn = kvs_get($remote_rank . "qpnum/" . $local_rank). + * a memory area to read from and write to HCA, + * and a memory area to read from HCA and write to DRAM is + * associated with each connection, so a rank should perform + * rkey = kvs_get($remote_rank . "rkey/" . $local_rank) + * and raddr = kvs_get($remote_rank . "raddr/" . $local_rank). */ + for (i = 0; i < MPID_nem_ib_nranks; i++) { + + /* lid and gid are common for all remote-ranks */ + if (i == 0) { + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &my_lid, + sizeof(uint16_t)); + dprintf("get_business_card,lid=%04x\n", my_lid); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, MPID_NEM_IB_LID_KEY, + (uint8_t *) & my_lid, sizeof(uint16_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_kvs_put_binary"); + + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &my_gid, + sizeof(union ibv_gid)); + + dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); + dprintf("get_business_card,sz=%ld,my_gid=", sizeof(union ibv_gid)); + for (j = 0; j < 16; j++) { + dprintf("%02x", (int) my_gid.raw[j]); + } + dprintf("\n"); + + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, MPID_NEM_IB_GID_KEY, + (uint8_t *) & my_gid, sizeof(union ibv_gid)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_kvs_put_binary"); + dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); + } + + /* we use different RDMA-rbuf for different senders. + * so announce like this: + * <"0/qpn/0", 0xa0000> + * <"0/qpn/1", 0xb0000> + * <"0/qpn/2", 0xc0000> + * <"0/qpn/3", 0xd0000> + */ + strcpy(key_str, MPID_NEM_IB_QPN_KEY); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); + +#if 0 + /* lmt-put */ + strcpy(key_str, MPID_NEM_IB_QPN_KEY); + strcat(key_str, "lmt-put"); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + ibcom_errno = + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[i].fd_lmt_put, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_qpnum, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); +#endif + + strcpy(key_str, MPID_NEM_IB_RMEM_KEY); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + + ibcom_errno = + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, MPID_NEM_IB_COM_INFOKEY_MR_ADDR, + &my_rmem, sizeof(void *)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); + + dprintf("rmem=%p\n", my_rmem); + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rmem, + sizeof(void *)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); + + strcpy(key_str, MPID_NEM_IB_RKEY_KEY); + sprintf(remote_rank_str, "/%x", i); + strcat(key_str, remote_rank_str); + + ibcom_errno = + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[i].fd, MPID_NEM_IB_COM_RDMAWR_TO, MPID_NEM_IB_COM_INFOKEY_MR_RKEY, + &my_rkey, sizeof(int)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); + + mpi_errno = + MPID_nem_ib_kvs_put_binary(MPID_nem_ib_myrank, key_str, (uint8_t *) & my_rkey, + sizeof(uint32_t)); + MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_kvs_put_binary"); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_ANNOUNCE_NETWORK_ADDR); + return mpi_errno; + fn_fail: + MPIU_CHKLMEM_FREEALL(); + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_connect_to_root +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPID_nem_ib_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc) +{ + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CONNECT_TO_ROOT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CONNECT_TO_ROOT); + + dprintf("toroot,%d->%d", MPID_nem_ib_myrank, new_vc->pg_rank); + /* not implemented */ + + //fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_CONNECT_TO_ROOT); + return MPI_SUCCESS; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_vc_init +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPID_nem_ib_vc_init(MPIDI_VC_t * vc) +{ + MPIDI_CH3I_VC *vc_ch = VC_CH(vc); + int mpi_errno = MPI_SUCCESS; + + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + int ibcom_errno; + size_t s; + MPID_nem_ib_conn_t *sc; + off_t offset; + + int remote_qpnum; + uint16_t remote_lid; + union ibv_gid remote_gid; + void *remote_rmem; + int remote_rkey; + + char key_str[256], remote_rank_str[256]; + + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_VC_INIT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_VC_INIT); + + vc_ib->sc = &MPID_nem_ib_conns[vc->pg_rank]; + + /* store pointer to MPID_nem_ib_com */ + ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd, &vc_ib->ibcom); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); + + ibcom_errno = + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_conns[vc->pg_rank].fd_lmt_put, &vc_ib->ibcom_lmt_put); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); + + //dprintf("vc_init,open,fd=%d,ptr=%p,rsr_seq_num_poll=%d\n", MPID_nem_ib_conns[vc->pg_rank].fd, vc_ib->MPID_nem_ib_com, vc_ib->ibcom->rsr_seq_num_poll); + + /* initialize sendq */ + vc_ib->sendq.head = NULL; + vc_ib->sendq.tail = NULL; + vc_ib->sendq_lmt_put.head = NULL; + vc_ib->sendq_lmt_put.tail = NULL; + + /* rank is sent as wr_id and used to obtain vc in poll */ + MPID_nem_ib_conns[vc->pg_rank].vc = vc; + MPIU_ERR_CHKANDJUMP(MPID_nem_ib_npollingset + 1 > MPID_NEM_IB_MAX_POLLINGSET, mpi_errno, + MPI_ERR_OTHER, "**MPID_nem_ib_npollingset"); + MPID_nem_ib_pollingset[MPID_nem_ib_npollingset++] = vc; + //printf("vc_init,%d->%d,vc=%p,npollingset=%d\n", MPID_nem_ib_myrank, vc->pg_rank, vc, MPID_nem_ib_npollingset); + + /* wait until you-to-me eager-send QP becomes RTR */ + MPID_nem_ib_com_t *MPID_nem_ib_com_scratch_pad; + ibcom_errno = + MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[vc->pg_rank], &MPID_nem_ib_com_scratch_pad); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); + + int ntrial = 0; + volatile MPID_nem_ib_com_qp_state_t *rstate = + (MPID_nem_ib_com_qp_state_t *) ((uint8_t *) MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + + vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t)); + dprintf("ib_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate)); + while (rstate->state != MPID_NEM_IB_COM_QP_STATE_RTR) { + __asm__ __volatile__("pause;":::"memory"); + if (++ntrial > 1024) { + /* detect RDMA-write failure */ + ibcom_errno = MPID_nem_ib_drain_scq_scratch_pad(); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_drain_scq_scratch_pad"); + } + } + dprintf("ib_init,you-to-me eager-send QP is RTR\n"); + + /* post MPID_NEM_IB_COM_MAX_SQ_CAPACITY of recv commands beforehand, replenish when retiring them in ib_poll */ + int i; + for (i = 0; i < MPID_NEM_IB_COM_MAX_RQ_CAPACITY; i++) { + //dprintf("irecv,%d->%d\n", MPID_nem_ib_myrank, vc->pg_rank); + ibcom_errno = MPID_nem_ib_com_irecv(MPID_nem_ib_conns[vc->pg_rank].fd, (uint64_t) vc->pg_rank); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_irecv"); + } + + MPIDI_CHANGE_VC_STATE(vc, ACTIVE); + + + uint32_t max_msg_sz; + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[vc->pg_rank].fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ, + &max_msg_sz, sizeof(max_msg_sz)); + VC_FIELD(vc, pending_sends) = 0; +#ifdef MPID_NEM_IB_ONDEMAND + VC_FIELD(vc, is_connected) = 0; +#endif + + MPIU_Assert(sizeof(sz_hdrmagic_t) == 8); /* assumption in ib_ibcom.h */ + MPIU_Assert(sizeof(tailmagic_t) == 1); /* assumption in ib_ibcom.h */ + + uint32_t sz; +#if 0 + /* assumption in released(), must be power of two */ + sz = MPID_NEM_IB_COM_RDMABUF_NSEG; + while ((sz & 1) == 0) { + sz >>= 1; + } + sz >>= 1; + if (sz) { + MPIU_Assert(0); + } +#endif + + /* assumption in ib_poll.c, must be power of two */ + for (sz = MPID_NEM_IB_COM_RDMABUF_SZSEG; sz > 0; sz >>= 1) { + if (sz != 1 && (sz & 1)) { + MPIU_Assert(0); + } + } + + char *val; + val = getenv("MP2_IBA_EAGER_THRESHOLD"); + vc->eager_max_msg_sz = val ? atoi(val) : MPID_NEM_IB_EAGER_MAX_MSG_SZ; + vc->ready_eager_max_msg_sz = val ? atoi(val) : MPID_NEM_IB_EAGER_MAX_MSG_SZ; + dprintf("ib_vc_init,vc->eager_max_msg_sz=%d\n", vc->eager_max_msg_sz); + + /* vc->rndvSend_fn is set in MPID_nem_vc_init (in src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c) */ + ; + vc->sendNoncontig_fn = MPID_nem_ib_SendNoncontig; + + vc->comm_ops = &comm_ops; + + + /* register packet handler */ + vc_ch->pkt_handler = MPID_nem_ib_pkt_handler; + vc_ch->num_pkt_handlers = MPIDI_NEM_IB_PKT_NUM_PKT_HANDLERS; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_EAGER_SEND] = MPID_nem_ib_PktHandler_EagerSend; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_PUT] = MPID_nem_ib_PktHandler_Put; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_GET] = MPID_nem_ib_PktHandler_Get; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_GET_RESP] = MPID_nem_ib_PktHandler_GetResp; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_ACCUMULATE] = MPID_nem_ib_PktHandler_Accumulate; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_LMT_GET_DONE] = MPID_nem_ib_pkt_GET_DONE_handler; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REQ_SEQ_NUM] = MPID_nem_ib_PktHandler_req_seq_num; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM] = + MPID_nem_ib_PktHandler_reply_seq_num; + MPID_nem_ib_pkt_handler[MPIDI_NEM_IB_PKT_CHG_RDMABUF_OCC_NOTIFY_STATE] = + MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state; + + /* register CH3 send/recv functions */ + vc_ch->iStartContigMsg = MPID_nem_ib_iStartContigMsg; + vc_ch->iSendContig = MPID_nem_ib_iSendContig; + + /* register CH3--lmt send/recv functions */ + vc_ch->lmt_initiate_lmt = MPID_nem_ib_lmt_initiate_lmt; + vc_ch->lmt_start_recv = MPID_nem_ib_lmt_start_recv; + vc_ch->lmt_handle_cookie = MPID_nem_ib_lmt_handle_cookie; + vc_ch->lmt_done_send = MPID_nem_ib_lmt_done_send; + vc_ch->lmt_done_recv = MPID_nem_ib_lmt_done_recv; + vc_ch->lmt_vc_terminated = MPID_nem_ib_lmt_vc_terminated; + vc_ch->next = NULL; + vc_ch->prev = NULL; + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_VC_INIT); + return mpi_errno; + fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_vc_destroy +#undef FCNAME +#define FCNAME MPIDI_QUOTE(FUNCNAME) +int MPID_nem_ib_vc_destroy(MPIDI_VC_t * vc) +{ + int mpi_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_VC_DESTROY); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_VC_DESTROY); + /* currently do nothing */ + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_VC_DESTROY); + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPID_nem_ib_vc_terminate +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc) +{ + dprintf("ib_vc_terminate,enter\n"); + int mpi_errno = MPI_SUCCESS; + int ibcom_errno; + int req_errno = MPI_SUCCESS; + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_VC_TERMINATE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_VC_TERMINATE); + + /* Check to make sure that it's OK to terminate the + * connection without making sure that all sends have been sent */ + /* it is safe to only check command queue because + * data transactions always proceed after confirming send by MPI_Wait + * and control transactions always proceed after receiveing reply */ + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + + dprintf("init,before,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", + MPID_nem_ib_myrank, vc->pg_rank, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, + pending_sends)); + + /* update remote RDMA-write-to buffer occupancy */ +#if 0 /* we can't send it when the other party has closed QP */ + while (MPID_nem_ib_diff32 + (vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent) > 0) { + MPID_nem_ib_send_reply_seq_num(vc); + } +#endif + + /* update local RDMA-write-to buffer occupancy */ +#if 0 + while (MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail) > 0) { + MPID_nem_ib_poll_eager(vc); + } +#endif + + /* drain sendq */ + while (!MPID_nem_ib_sendq_empty(vc_ib->sendq)) { + MPID_nem_ib_send_progress(vc_ib); + } + + dprintf("init,middle,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", + MPID_nem_ib_myrank, vc->pg_rank, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, + pending_sends)); + + if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { + usleep(1000); + MPID_nem_ib_drain_scq(0); + } + dprintf("init,middle2,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", + MPID_nem_ib_myrank, vc->pg_rank, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, + pending_sends)); + + if (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { + usleep(1000); + MPID_nem_ib_drain_scq(0); + } +#if 0 + /* drain scq */ + while (MPID_nem_ib_ncqe > 0 || VC_FIELD(vc, pending_sends) > 0) { + usleep(1000); + MPID_nem_ib_drain_scq(0); + //printf("%d\n", VC_FIELD(vc, pending_sends)); + //printf("%d\n", MPID_nem_ib_ncqe); + } +#endif + + dprintf("init,after ,%d->%d,r rdmaocc=%d,l rdmaocc=%d,sendq=%d,ncqe=%d,pending_sends=%d\n", + MPID_nem_ib_myrank, vc->pg_rank, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), + MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, + pending_sends)); + + /* drain scratch-pad scq */ + ibcom_errno = MPID_nem_ib_drain_scq_scratch_pad(); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, + "**MPID_nem_ib_drain_scq_scratch_pad"); + + mpi_errno = MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED); + if (mpi_errno) { + MPIU_ERR_POP(mpi_errno); + } + + fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_VC_TERMINATE); + return mpi_errno; + fn_fail: + MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "failure. mpi_errno = %d", mpi_errno)); + goto fn_exit; + +} diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c similarity index 70% rename from src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c rename to src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c index 585ba27..4552f51 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c @@ -5,13 +5,13 @@ * See COPYRIGHT in top-level directory. */ -#include "dcfa_impl.h" +#include "ib_impl.h" -//#define DEBUG_DCFA_LMT +//#define MPID_NEM_IB_DEBUG_LMT #ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ #undef dprintf #endif -#ifdef DEBUG_DCFA_LMT +#ifdef MPID_NEM_IB_DEBUG_LMT #define dprintf printf #else #define dprintf(...) @@ -19,10 +19,10 @@ /* Get mode: sender sends RTS */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_initiate_lmt +#define FUNCNAME MPID_nem_ib_lmt_initiate_lmt #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, +int MPID_nem_ib_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, struct MPID_Request *req) { int mpi_errno = MPI_SUCCESS; @@ -31,12 +31,12 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts MPID_Datatype *dt_ptr; MPI_Aint dt_true_lb; MPIDI_CH3I_VC *vc_ch = VC_CH(vc); - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_INITIATE_LMT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_INITIATE_LMT); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_INITIATE_LMT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_INITIATE_LMT); - dprintf("lmt_initiate_lmt,enter,%d->%d,req=%p\n", MPID_nem_dcfa_myrank, vc->pg_rank, req); + dprintf("lmt_initiate_lmt,enter,%d->%d,req=%p\n", MPID_nem_ib_myrank, vc->pg_rank, req); /* obtain dt_true_lb */ /* see MPIDI_Datatype_get_info(in, in, out, out, out, out) (in src/mpid/ch3/include/mpidimpl.h) */ @@ -44,8 +44,8 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts dt_true_lb); /* malloc memory area for cookie. auto variable is NG because isend does not copy payload */ - MPID_nem_dcfa_lmt_cookie_t *s_cookie_buf = - (MPID_nem_dcfa_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_dcfa_lmt_cookie_t)); + MPID_nem_ib_lmt_cookie_t *s_cookie_buf = + (MPID_nem_ib_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_lmt_cookie_t)); /* remember address to "free" when receiving DONE from receiver */ req->ch.s_cookie = s_cookie_buf; @@ -97,20 +97,20 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts MPIU_Assert(((MPID_nem_pkt_lmt_rts_t *) rts_pkt)->data_sz == data_sz); s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); /* prepare magic */ - //*((uint32_t*)(write_from_buf + data_sz - sizeof(tailmagic_t))) = IBCOM_MAGIC; + //*((uint32_t*)(write_from_buf + data_sz - sizeof(tailmagic_t))) = MPID_NEM_IB_COM_MAGIC; #if 1 /* embed RDMA-write-to buffer occupancy information */ - dprintf("lmt_initiate_lmt,rsr_seq_num_tail=%d\n", vc_dcfa->ibcom->rsr_seq_num_tail); + dprintf("lmt_initiate_lmt,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); /* embed RDMA-write-to buffer occupancy information */ - s_cookie_buf->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; + s_cookie_buf->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; /* remember the last one sent */ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = vc_dcfa->ibcom->rsr_seq_num_tail; + vc_ib->ibcom->rsr_seq_num_tail_last_sent = vc_ib->ibcom->rsr_seq_num_tail; #endif /* put IB rkey */ - struct ibv_mr *mr = ibcom_reg_mr_fetch(write_from_buf, data_sz); - MPIU_ERR_CHKANDJUMP(!mr, mpi_errno, MPI_ERR_OTHER, "**ibcom_reg_mr_fetch"); + struct ibv_mr *mr = MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz); + MPIU_ERR_CHKANDJUMP(!mr, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch"); #ifdef HAVE_LIBDCFA s_cookie_buf->addr = (void *) mr->host_addr; dprintf("lmt_initiate_lmt,s_cookie_buf->addr=%p\n", s_cookie_buf->addr); @@ -122,10 +122,10 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts s_cookie_buf->addr, s_cookie_buf->rkey); /* send cookie. rts_pkt as the MPI-header, s_cookie_buf as the payload */ MPID_nem_lmt_send_RTS(vc, (MPID_nem_pkt_lmt_rts_t *) rts_pkt, s_cookie_buf, - sizeof(MPID_nem_dcfa_lmt_cookie_t)); + sizeof(MPID_nem_ib_lmt_cookie_t)); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_INITIATE_LMT); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_INITIATE_LMT); return mpi_errno; fn_fail: goto fn_exit; @@ -133,49 +133,49 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts /* essential lrecv part extracted for dequeueing and issue from sendq */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_start_recv_core +#define FUNCNAME MPID_nem_ib_lmt_start_recv_core #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, +int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, void *write_to_buf) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; struct MPIDI_VC *vc = req->ch.vc; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_START_RECV_CORE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_START_RECV_CORE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE); ibcom_errno = - ibcom_lrecv(vc_dcfa->sc->fd, (uint64_t) req, raddr, req->ch.lmt_data_sz, rkey, + MPID_nem_ib_com_lrecv(vc_ib->sc->fd, (uint64_t) req, raddr, req->ch.lmt_data_sz, rkey, write_to_buf); - MPID_nem_dcfa_ncqe += 1; - //dprintf("start_recv,ncqe=%d\n", MPID_nem_dcfa_ncqe); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_lrecv"); - dprintf("lmt_start_recv_core,MPID_nem_dcfa_ncqe=%d\n", MPID_nem_dcfa_ncqe); + MPID_nem_ib_ncqe += 1; + //dprintf("start_recv,ncqe=%d\n", MPID_nem_ib_ncqe); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lrecv"); + dprintf("lmt_start_recv_core,MPID_nem_ib_ncqe=%d\n", MPID_nem_ib_ncqe); dprintf ("lmt_start_recv_core,req=%p,sz=%ld,write_to_buf=%p,lmt_pack_buf=%p,user_buf=%p,raddr=%p,rkey=%08x,tail=%p=%02x\n", req, req->ch.lmt_data_sz, write_to_buf, REQ_FIELD(req, lmt_pack_buf), req->dev.user_buf, raddr, rkey, write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t), *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t)))); -#ifdef LMT_GET_CQE - MPID_nem_dcfa_ncqe_to_drain += 1; /* use CQE instead of polling */ +#ifdef MPID_NEM_IB_LMT_GET_CQE + MPID_nem_ib_ncqe_to_drain += 1; /* use CQE instead of polling */ #else - /* drain_scq and dcfa_poll is not ordered, so both can decrement ref_count */ + /* drain_scq and ib_poll is not ordered, so both can decrement ref_count */ MPIR_Request_add_ref(req); - /* register to poll list in dcfa_poll() */ + /* register to poll list in ib_poll() */ /* don't use req->dev.next because it causes unknown problem */ - MPID_nem_dcfa_lmtq_enqueue(&MPID_nem_dcfa_lmtq, req); + MPID_nem_ib_lmtq_enqueue(&MPID_nem_ib_lmtq, req); dprintf("lmt_start_recv_core,lmtq enqueue\n"); //volatile uint8_t* tailmagic = (uint8_t*)((void*)req->dev.user_buf + req->ch.lmt_data_sz - sizeof(uint8_t)); //dprintf("start_recv_core,cur_tail=%02x,lmt_receiver_tail=%02x\n", *tailmagic, REQ_FIELD(req, lmt_receiver_tail)); #endif fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_START_RECV_CORE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE); return mpi_errno; fn_fail: goto fn_exit; @@ -186,10 +186,10 @@ int MPID_nem_dcfa_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uin caller: (in mpid_nem_lmt.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_start_recv +#define FUNCNAME MPID_nem_ib_lmt_start_recv #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie) +int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -198,21 +198,21 @@ int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_Datatype *dt_ptr; MPI_Aint dt_true_lb; MPIDI_CH3I_VC *vc_ch = VC_CH(vc); - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_START_RECV); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_START_RECV); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_START_RECV); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_START_RECV); - dprintf("lmt_start_recv,enter,%d<-%d,req=%p\n", MPID_nem_dcfa_myrank, vc->pg_rank, req); + dprintf("lmt_start_recv,enter,%d<-%d,req=%p\n", MPID_nem_ib_myrank, vc->pg_rank, req); /* obtain dt_true_lb */ /* see MPIDI_Datatype_get_info(in, in, out, out, out, out) (in src/mpid/ch3/include/mpidimpl.h) */ MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb); - MPID_nem_dcfa_lmt_cookie_t *s_cookie_buf = s_cookie.iov_base; + MPID_nem_ib_lmt_cookie_t *s_cookie_buf = s_cookie.iov_base; - /* stash vc for dcfa_poll */ + /* stash vc for ib_poll */ req->ch.vc = vc; void *write_to_buf; @@ -221,13 +221,13 @@ int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, } else { //REQ_FIELD(req, lmt_pack_buf) = MPIU_Malloc((size_t)req->ch.lmt_data_sz); - REQ_FIELD(req, lmt_pack_buf) = MPID_nem_dcfa_stmalloc((size_t) req->ch.lmt_data_sz); + REQ_FIELD(req, lmt_pack_buf) = MPID_nem_ib_stmalloc((size_t) req->ch.lmt_data_sz); MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER, "**outofmemory"); write_to_buf = REQ_FIELD(req, lmt_pack_buf); } -#ifdef LMT_GET_CQE +#ifdef MPID_NEM_IB_LMT_GET_CQE #else /* unmark magic */ *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))) = ~s_cookie_buf->tail; /* size in cookie was not set */ @@ -243,14 +243,14 @@ int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t), *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t)))); - //dprintf("lmt_start_recv,sendq_empty=%d,ncom=%d,ncqe=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY); + //dprintf("lmt_start_recv,sendq_empty=%d,ncom=%d,ncqe=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY); /* try to issue RDMA-read command */ int slack = 1; /* slack for control packet bringing sequence number */ - if (MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq) && - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY - slack && - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY - slack) { - mpi_errno = MPID_nem_dcfa_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, write_to_buf); /* fast path not storing raddr and rkey */ + if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) { + mpi_errno = MPID_nem_ib_lmt_start_recv_core(req, s_cookie_buf->addr, s_cookie_buf->rkey, write_to_buf); /* fast path not storing raddr and rkey */ if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -258,51 +258,51 @@ int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, else { /* enqueue command into send_queue */ dprintf("lmt_start_recv, enqueuing,sendq_empty=%d,ncom=%d,ncqe=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY); + MPID_nem_ib_sendq_empty(vc_ib->sendq), + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY); /* make raddr, (sz is in rreq->ch.lmt_data_sz), rkey, (user_buf is in req->dev.user_buf) survive enqueue, free cookie, dequeue */ REQ_FIELD(req, lmt_raddr) = s_cookie_buf->addr; REQ_FIELD(req, lmt_rkey) = s_cookie_buf->rkey; REQ_FIELD(req, lmt_write_to_buf) = write_to_buf; - MPID_nem_dcfa_sendq_enqueue(&vc_dcfa->sendq, req); + MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, req); } /* extract embeded RDMA-write-to buffer occupancy information */ dprintf("lmt_start_recv,old lsr_seq_num=%d,s_cookie_buf->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, s_cookie_buf->seq_num_tail); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, s_cookie_buf->seq_num_tail); - //dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_dcfa->ibcom->lsr_seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail, s_cookie_buf->seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, s_cookie_buf->seq_num_tail); + //dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail); -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, - &vc_dcfa->ibcom->lsr_seq_num_tail); - //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, + &vc_ib->ibcom->lsr_seq_num_tail); + //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif - //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ //dprintf("lmt_start_recv,reply_seq_num,send_progress\n"); - if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)) { + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq)) { dprintf("lmt_start_recv,ncom=%d,ncqe=%d,diff=%d\n", - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG); + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG); } - if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq) && - MPID_nem_dcfa_sendq_ready_to_send_head(vc_dcfa)) { + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && + MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) { dprintf("lmt_start_recv,send_progress\n"); fflush(stdout); - MPID_nem_dcfa_send_progress(vc_dcfa); + MPID_nem_ib_send_progress(vc_ib); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_START_RECV); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_START_RECV); return mpi_errno; fn_fail: goto fn_exit; @@ -310,10 +310,10 @@ int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, /* fall-back to lmt-get if end-flag of send-buf has the same value as the end-flag of recv-buf */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_switch_send +#define FUNCNAME MPID_nem_ib_lmt_switch_send #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) +int MPID_nem_ib_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) { int mpi_errno = MPI_SUCCESS; int dt_contig; @@ -321,10 +321,10 @@ int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) MPID_Datatype *dt_ptr; MPI_Aint dt_true_lb; MPID_IOV r_cookie = req->ch.lmt_tmp_cookie; - MPID_nem_dcfa_lmt_cookie_t *r_cookie_buf = r_cookie.iov_base; + MPID_nem_ib_lmt_cookie_t *r_cookie_buf = r_cookie.iov_base; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_SWITCH_SEND); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_SWITCH_SEND); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND); MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb); @@ -374,7 +374,7 @@ int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) #endif fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_SWITCH_SEND); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_SWITCH_SEND); return mpi_errno; fn_fail: goto fn_exit; @@ -382,21 +382,21 @@ int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) /* when cookie is received in the middle of the lmt */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_handle_cookie +#define FUNCNAME MPID_nem_ib_lmt_handle_cookie #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie) +int MPID_nem_ib_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie) { int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_HANDLE_COOKIE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_HANDLE_COOKIE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_HANDLE_COOKIE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_HANDLE_COOKIE); dprintf("lmt_handle_cookie,enter\n"); /* Nothing to do */ fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_HANDLE_COOKIE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_HANDLE_COOKIE); return mpi_errno; //fn_fail: goto fn_exit; @@ -404,18 +404,18 @@ int MPID_nem_dcfa_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *re /* when sender receives DONE from receiver */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_done_send +#define FUNCNAME MPID_nem_ib_lmt_done_send #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req) +int MPID_nem_ib_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req) { int mpi_errno = MPI_SUCCESS; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_SEND); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_SEND); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_DONE_SEND); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_DONE_SEND); dprintf("lmt_done_send,enter,%d<-%d,req=%p,REQ_FIELD(req, lmt_pack_buf)=%p\n", - MPID_nem_dcfa_myrank, vc->pg_rank, req, REQ_FIELD(req, lmt_pack_buf)); + MPID_nem_ib_myrank, vc->pg_rank, req, REQ_FIELD(req, lmt_pack_buf)); /* free memory area for cookie */ @@ -460,14 +460,14 @@ int MPID_nem_dcfa_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req) /* mark completion on sreq */ MPIU_ERR_CHKANDJUMP(req->dev.OnDataAvail, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_lmt_done_send"); + "**MPID_nem_ib_lmt_done_send"); dprintf("lmt_done_send,1,req=%p,pcc=%d\n", req, MPIDI_CH3I_progress_completion_count.v); MPIDI_CH3U_Request_complete(req); dprintf("lmt_done_send,2,req=%p,pcc=%d\n", req, MPIDI_CH3I_progress_completion_count.v); //dprintf("lmt_done_send, mark completion on sreq\n"); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_SEND); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_DONE_SEND); return mpi_errno; fn_fail: goto fn_exit; @@ -475,18 +475,18 @@ int MPID_nem_dcfa_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req) /* lmt-put (1) sender sends done when finding cqe of put (2) packet-handler of DONE on receiver (3) here */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_done_recv +#define FUNCNAME MPID_nem_ib_lmt_done_recv #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq) +int MPID_nem_ib_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq) { int mpi_errno = MPI_SUCCESS; MPIDI_CH3I_VC *vc_ch = VC_CH(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_RECV); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_RECV); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_DONE_RECV); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_DONE_RECV); - dprintf("lmt_done_recv,enter,rreq=%p,head=%p\n", rreq, MPID_nem_dcfa_lmtq.head); + dprintf("lmt_done_recv,enter,rreq=%p,head=%p\n", rreq, MPID_nem_ib_lmtq.head); int is_contig; @@ -511,12 +511,12 @@ int MPID_nem_dcfa_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq) MPIR_STATUS_SET_COUNT(rreq->status, last); rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, - MPI_ERR_TYPE, "**MPID_nem_dcfa_lmt_done_recv", 0); + MPI_ERR_TYPE, "**MPID_nem_ib_lmt_done_recv", 0); /* --END ERROR HANDLING-- */ } //MPIU_Free(REQ_FIELD(rreq, lmt_pack_buf)); - MPID_nem_dcfa_stfree(REQ_FIELD(rreq, lmt_pack_buf), (size_t) rreq->ch.lmt_data_sz); + MPID_nem_ib_stfree(REQ_FIELD(rreq, lmt_pack_buf), (size_t) rreq->ch.lmt_data_sz); } dprintf("lmt_done_recv,1,req=%p,pcc=%d\n", rreq, MPIDI_CH3I_progress_completion_count.v); @@ -524,28 +524,28 @@ int MPID_nem_dcfa_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq) dprintf("lmt_done_recv,2,pcc=%d\n", MPIDI_CH3I_progress_completion_count.v); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_RECV); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_DONE_RECV); return mpi_errno; //fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_lmt_vc_terminated +#define FUNCNAME MPID_nem_ib_lmt_vc_terminated #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_lmt_vc_terminated(struct MPIDI_VC *vc) +int MPID_nem_ib_lmt_vc_terminated(struct MPIDI_VC *vc) { int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_LMT_VC_TERMINATED); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_LMT_VC_TERMINATED); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_VC_TERMINATED); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_VC_TERMINATED); dprintf("lmt_vc_terminated,enter\n"); /* Nothing to do */ fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_VC_TERMINATED); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_VC_TERMINATED); return mpi_errno; //fn_fail: goto fn_exit; diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c similarity index 56% rename from src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c rename to src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c index 873c671..c6cd009 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c @@ -5,13 +5,13 @@ * See COPYRIGHT in top-level directory. */ -#include "dcfa_impl.h" +#include "ib_impl.h" -//#define DEBUG_DCFA_POLL +//#define MPID_NEM_IB_DEBUG_POLL #ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ #undef dprintf #endif -#ifdef DEBUG_DCFA_POLL +#ifdef MPID_NEM_IB_DEBUG_POLL #define dprintf printf #else #define dprintf(...) @@ -20,40 +20,40 @@ static int entered_drain_scq = 0; #if 0 -#define MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET MPID_nem_dcfa_send_progress(vc_dcfa); +#define MPID_NEM_IB_SEND_PROGRESS_POLLINGSET MPID_nem_ib_send_progress(vc_ib); #else -#define MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET { \ +#define MPID_NEM_IB_SEND_PROGRESS_POLLINGSET { \ int n; \ - for(n = 0; n < MPID_nem_dcfa_npollingset; n++) { \ - MPIDI_VC_t *vc_n = MPID_nem_dcfa_pollingset[n]; \ - /*MPID_nem_dcfa_debug_current_vc_dcfa = vc_dcfa;*/ \ - MPID_nem_dcfa_send_progress(VC_DCFA(vc_n)); \ + for(n = 0; n < MPID_nem_ib_npollingset; n++) { \ + MPIDI_VC_t *vc_n = MPID_nem_ib_pollingset[n]; \ + /*MPID_nem_ib_debug_current_vc_ib = vc_ib;*/ \ + MPID_nem_ib_send_progress(VC_IB(vc_n)); \ } \ } #endif #if 1 -#define MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS \ - if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq) && MPID_nem_dcfa_sendq_ready_to_send_head(vc_dcfa)) { \ - MPID_nem_dcfa_send_progress(vc_dcfa); \ +#define MPID_NEM_IB_CHECK_AND_SEND_PROGRESS \ + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) { \ + MPID_nem_ib_send_progress(vc_ib); \ } #else -#define MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET +#define MPID_NEM_IB_CHECK_AND_SEND_PROGRESS MPID_NEM_IB_SEND_PROGRESS_POLLINGSET #endif #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_drain_scq +#define FUNCNAME MPID_nem_ib_drain_scq #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_drain_scq(int dont_call_progress) +int MPID_nem_ib_drain_scq(int dont_call_progress) { int mpi_errno = MPI_SUCCESS; int result; int i; - struct ibv_wc cqe[IBCOM_MAX_CQ_HEIGHT_DRAIN]; + struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ); /* prevent a call path drain_scq -> send_progress -> drain_scq */ if (entered_drain_scq) { @@ -63,12 +63,12 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) entered_drain_scq = 1; #if 0 /*def HAVE_LIBDCFA */ - result = ibv_poll_cq(rc_shared_scq, 1, &cqe[0]); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, 1, &cqe[0]); #else - result = ibv_poll_cq(rc_shared_scq, /*3 */ IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq, /*3 */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); #endif - MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,dcfa,ibv_poll_cq"); + MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { dprintf("poll,scq,result=%d\n", result); @@ -90,7 +90,7 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) dprintf("drain_scq,req=%p,req->ref_count=%d,cc_ptr=%d\n", req, req->ref_count, *req->cc_ptr); if (req->ref_count <= 0) { - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(req->ch.vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); printf("%d\n", *(int *) 0); } @@ -106,7 +106,7 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) } #endif MPIU_ERR_CHKANDJUMP(cqe[i].status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq"); + "**MPID_nem_ib_drain_scq"); /* * packets generated by MPIDI_CH3_iStartMsgv has req_type of RECV @@ -120,16 +120,16 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) dprintf("drain_scq,send/recv,eager,req_type=%d,,comm=%p,opcode=%d\n", req_type, req->comm, cqe[i].opcode); - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(req->ch.vc); - dprintf("drain_scq,MPIDI_REQUEST_EAGER_MSG,%d->%d,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_myrank, req->ch.vc->pg_rank, MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); /* moved before MPID_Request_release because this references req->ch.vc */ + MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); + dprintf("drain_scq,MPIDI_REQUEST_EAGER_MSG,%d->%d,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_myrank, req->ch.vc->pg_rank, MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* moved before MPID_Request_release because this references req->ch.vc */ /* free temporal buffer for eager-send non-contiguous data. * MPIDI_Request_create_sreq (in mpid_isend.c) sets req->dev.datatype * control message has a req_type of MPIDI_REQUEST_TYPE_RECV and * msg_type of MPIDI_REQUEST_EAGER_MSG because * control message send follows - * MPIDI_CH3_iStartMsg/v-->MPID_nem_dcfa_iStartContigMsg-->MPID_nem_dcfa_iSendContig - * and MPID_nem_dcfa_iSendContig set req->dev.state to zero. + * MPIDI_CH3_iStartMsg/v-->MPID_nem_ib_iStartContigMsg-->MPID_nem_ib_iSendContig + * and MPID_nem_ib_iSendContig set req->dev.state to zero. * see MPID_Request_create (in src/mpid/ch3/src/ch3u_request.c) * eager-short message has req->comm of zero */ @@ -146,16 +146,16 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) } /* decrement the number of entries in IB command queue */ - vc_dcfa->ibcom->ncom -= 1; - MPID_nem_dcfa_ncqe -= 1; - dprintf("drain_scq,eager-send,ncqe=%d\n", MPID_nem_dcfa_ncqe); + vc_ib->ibcom->ncom -= 1; + MPID_nem_ib_ncqe -= 1; + dprintf("drain_scq,eager-send,ncqe=%d\n", MPID_nem_ib_ncqe); MPIU_Assert(req->ref_count == 1 || req->ref_count == 2); /* ref_count is decremented in drain_scq and wait */ if (*req->cc_ptr > 0) { - dprintf("drain_scq,MPID_nem_dcfa_ncqe_nces=%d,cc_ptr=%d,pending_sends=%d\n", - MPID_nem_dcfa_ncqe_nces, *req->cc_ptr, VC_FIELD(req->ch.vc, pending_sends)); - MPID_nem_dcfa_ncqe_nces -= 1; + dprintf("drain_scq,MPID_nem_ib_ncqe_nces=%d,cc_ptr=%d,pending_sends=%d\n", + MPID_nem_ib_ncqe_nces, *req->cc_ptr, VC_FIELD(req->ch.vc, pending_sends)); + MPID_nem_ib_ncqe_nces -= 1; int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); @@ -184,16 +184,16 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) MPID_Request_release(req); } /* try to send from sendq */ - //dprintf("dcfa_poll,SCQ,!lmt,send_progress\n"); - if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)) { + //dprintf("ib_poll,SCQ,!lmt,send_progress\n"); + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq)) { dprintf("drain_scq,eager-send,ncom=%d,ncqe=%d,diff=%d\n", - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) < - IBCOM_RDMABUF_NSEG); + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) < + MPID_NEM_IB_COM_RDMABUF_NSEG); - MPID_Request *sreq = MPID_nem_dcfa_sendq_head(vc_dcfa->sendq); + MPID_Request *sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); int msg_type_sreq = MPIDI_Request_get_msg_type(sreq); if (sreq->kind == MPID_REQUEST_SEND && msg_type_sreq == MPIDI_REQUEST_EAGER_MSG) { @@ -206,14 +206,14 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) dprintf("drain_scq,eager-send,head is lmt RDMA-write\n"); } } - /* call MPID_nem_dcfa_send_progress for all VCs in polling-set + /* call MPID_nem_ib_send_progress for all VCs in polling-set * instead of VC which releases CQ, command * when releasing them * because commands for VC-A are blocked by the command * for VC-B and waiting in the sendq */ dprintf("drain_scq,eager-send,send_progress\n"); - //MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET; + //MPID_NEM_IB_SEND_PROGRESS_POLLINGSET; dprintf("drain_scq,eager-send,next\n"); @@ -222,19 +222,19 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) dprintf("drain_scq,GET_RESP,eager,req_type=%d,,comm=%p,opcode=%d\n", req_type, req->comm, cqe[i].opcode); - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(req->ch.vc); - dprintf("drain_scq,MPIDI_REQUEST_EAGER_MSG,%d->%d,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_myrank, req->ch.vc->pg_rank, MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); /* moved before MPID_Request_release because this references req->ch.vc */ + MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); + dprintf("drain_scq,MPIDI_REQUEST_EAGER_MSG,%d->%d,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_myrank, req->ch.vc->pg_rank, MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* moved before MPID_Request_release because this references req->ch.vc */ /* decrement the number of entries in IB command queue */ - vc_dcfa->ibcom->ncom -= 1; - MPID_nem_dcfa_ncqe -= 1; - dprintf("drain_scq,GET_RESP,ncqe=%d\n", MPID_nem_dcfa_ncqe); + vc_ib->ibcom->ncom -= 1; + MPID_nem_ib_ncqe -= 1; + dprintf("drain_scq,GET_RESP,ncqe=%d\n", MPID_nem_ib_ncqe); MPIU_Assert(req->ref_count == 1 || req->ref_count == 2); /* ref_count is decremented in drain_scq and wait */ - dprintf("drain_scq,MPID_nem_dcfa_ncqe_nces=%d,cc_ptr=%d,pending_sends=%d\n", - MPID_nem_dcfa_ncqe_nces, *req->cc_ptr, VC_FIELD(req->ch.vc, pending_sends)); - MPID_nem_dcfa_ncqe_nces -= 1; + dprintf("drain_scq,MPID_nem_ib_ncqe_nces=%d,cc_ptr=%d,pending_sends=%d\n", + MPID_nem_ib_ncqe_nces, *req->cc_ptr, VC_FIELD(req->ch.vc, pending_sends)); + MPID_nem_ib_ncqe_nces -= 1; int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); @@ -260,7 +260,7 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) MPIU_Assert(complete == TRUE); } - //MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET; + //MPID_NEM_IB_SEND_PROGRESS_POLLINGSET; dprintf("drain_scq,GET_RESP,next\n"); @@ -276,8 +276,8 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) dprintf("drain_scq,recv,rndv,rdma-read,kind=%d,opcode=%d\n", kind, cqe[i].opcode); - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(req->ch.vc); -#if defined(LMT_GET_CQE) + MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); +#if defined(MPID_NEM_IB_LMT_GET_CQE) /* unpack non-contiguous dt */ int is_contig; @@ -303,29 +303,29 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) MPIR_STATUS_SET_COUNT(req->status, last); req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, - MPI_ERR_TYPE, "**MPID_nem_dcfa_poll", 0); + MPI_ERR_TYPE, "**MPID_nem_ib_poll", 0); /* --END ERROR HANDLING-- */ } dprintf("drain_scq,lmt,GET_CQE,ref_count=%d,lmt_pack_buf=%p\n", req->ref_count, REQ_FIELD(req, lmt_pack_buf)); - MPID_nem_dcfa_stfree(REQ_FIELD(req, lmt_pack_buf), (size_t) req->ch.lmt_data_sz); + MPID_nem_ib_stfree(REQ_FIELD(req, lmt_pack_buf), (size_t) req->ch.lmt_data_sz); } dprintf("drain_scq,lmt,GET_CQE,lmt_send_GET_DONE,rsr_seq_num_tail=%d\n", - vc_dcfa->ibcom->rsr_seq_num_tail); + vc_ib->ibcom->rsr_seq_num_tail); - /* send done to sender. vc is stashed in MPID_nem_dcfa_lmt_start_recv (in dcfa_lmt.c) */ - MPID_nem_dcfa_lmt_send_GET_DONE(req->ch.vc, req); + /* send done to sender. vc is stashed in MPID_nem_ib_lmt_start_recv (in ib_lmt.c) */ + MPID_nem_ib_lmt_send_GET_DONE(req->ch.vc, req); #endif /* unmark "lmt is going on" */ - //dprintf("dcfa_poll,SCQ,lmt,%d->%d,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_myrank, req->ch.vc->pg_rank, MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); /* moved before MPID_Request_release because this references req->ch.vc */ + //dprintf("ib_poll,SCQ,lmt,%d->%d,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_myrank, req->ch.vc->pg_rank, MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* moved before MPID_Request_release because this references req->ch.vc */ /* decrement the number of entries in IB command queue */ - vc_dcfa->ibcom->ncom -= 1; - MPID_nem_dcfa_ncqe -= 1; - dprintf("drain_scq,rdma-read,ncqe=%d\n", MPID_nem_dcfa_ncqe); + vc_ib->ibcom->ncom -= 1; + MPID_nem_ib_ncqe -= 1; + dprintf("drain_scq,rdma-read,ncqe=%d\n", MPID_nem_ib_ncqe); -#ifdef LMT_GET_CQE +#ifdef MPID_NEM_IB_LMT_GET_CQE dprintf("drain_scq,GET_CQE,Request_complete\n"); /* mark completion on rreq */ MPIDI_CH3U_Request_complete(req); @@ -339,26 +339,26 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) REQ_FIELD(req, lmt_pack_buf)); /* debug, polling waits forever when freeing here. */ //free(REQ_FIELD(req, lmt_pack_buf)); - //MPID_nem_dcfa_stfree(REQ_FIELD(req, lmt_pack_buf), (size_t)req->ch.lmt_data_sz); - //dprintf("drain_scq,lmt,insert to free-list=%p\n", MPID_nem_dcfa_fl); + //MPID_nem_ib_stfree(REQ_FIELD(req, lmt_pack_buf), (size_t)req->ch.lmt_data_sz); + //dprintf("drain_scq,lmt,insert to free-list=%p\n", MPID_nem_ib_fl); //} else { //dprintf("drain_scq,GET&&!GET_CQE,ref_count=%d,lmt_pack_buf=%p\n", req->ref_count, REQ_FIELD(req, lmt_pack_buf)); //} } /* lmt_start_recv increments ref_count - * drain_scq and dcfa_poll is not ordered, so both can decrement ref_count */ + * drain_scq and ib_poll is not ordered, so both can decrement ref_count */ MPID_Request_release(req); #endif /* try to send from sendq */ - if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)) { + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq)) { dprintf("drain_scq,GET,ncom=%d,ncqe=%d,diff=%d\n", - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) < - IBCOM_RDMABUF_NSEG); - MPID_Request *sreq = MPID_nem_dcfa_sendq_head(vc_dcfa->sendq); + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) < + MPID_NEM_IB_COM_RDMABUF_NSEG); + MPID_Request *sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); int msg_type_sreq = MPIDI_Request_get_msg_type(sreq); if (sreq->kind == MPID_REQUEST_SEND && msg_type_sreq == MPIDI_REQUEST_EAGER_MSG) { @@ -368,54 +368,54 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) dprintf("drain_scq,eager-send,head is lmt\n"); } } - //if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq) && MPID_nem_dcfa_sendq_ready_to_send_head(vc_dcfa)) { + //if (!MPID_nem_ib_sendq_empty(vc_ib->sendq) && MPID_nem_ib_sendq_ready_to_send_head(vc_ib)) { dprintf("drain_scq,GET,send_progress\n"); fflush(stdout); - //MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET + //MPID_NEM_IB_SEND_PROGRESS_POLLINGSET //} } else { printf("drain_scq,unknown kind=%d,req_type=%d,msg_type=%d\n", kind, req_type, msg_type); assert(0); #if 1 // lazy consulting of completion queue - MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_drain_scq"); + MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); #else //printf("kind=%d\n", kind); #endif } } if (!dont_call_progress) { - MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET; + MPID_NEM_IB_SEND_PROGRESS_POLLINGSET; } fn_exit: entered_drain_scq = 0; - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_drain_scq_lmt_put +#define FUNCNAME MPID_nem_ib_drain_scq_lmt_put #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_drain_scq_lmt_put() +int MPID_nem_ib_drain_scq_lmt_put() { int mpi_errno = MPI_SUCCESS; int result; int i; - struct ibv_wc cqe[IBCOM_MAX_CQ_HEIGHT_DRAIN]; + struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_LMT_PUT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_LMT_PUT); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ_LMT_PUT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ_LMT_PUT); #if 0 /*def HAVE_LIBDCFA */ - result = ibv_poll_cq(rc_shared_scq_lmt_put, 1, &cqe[0]); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, 1, &cqe[0]); #else - result = ibv_poll_cq(rc_shared_scq_lmt_put, IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); #endif - MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,dcfa,ibv_poll_cq"); + MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { dprintf("drain_scq_lmt_put,found,result=%d\n", result); @@ -437,7 +437,7 @@ int MPID_nem_dcfa_drain_scq_lmt_put() } #endif MPIU_ERR_CHKANDJUMP(cqe[i].status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq_lmt_put"); + "**MPID_nem_ib_drain_scq_lmt_put"); /* Obtain sreq */ req = (MPID_Request *) cqe[i].wr_id; @@ -452,7 +452,7 @@ int MPID_nem_dcfa_drain_scq_lmt_put() if (req_type == MPIDI_REQUEST_TYPE_RECV && msg_type == MPIDI_REQUEST_RNDV_MSG) { /* lmt-put */ /* MPIDI_Request_set_type is not performed when - * MPID_Isend --> FDU_or_AEP --> recv_posted --> dcfa_poll --> PUTCTS packet-handler */ + * MPID_Isend --> FDU_or_AEP --> recv_posted --> ib_poll --> PUTCTS packet-handler */ dprintf("drain_scq_lmt_put,lmt-put found\n"); @@ -460,10 +460,10 @@ int MPID_nem_dcfa_drain_scq_lmt_put() #endif /* decrement the number of entries in IB command queue */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(req->ch.vc); - vc_dcfa->ibcom->ncom_lmt_put -= 1; - MPID_nem_dcfa_ncqe_lmt_put -= 1; - dprintf("drain_scq_lmt_put,rndv,ncqe=%d\n", MPID_nem_dcfa_ncqe_lmt_put); /*suspicious */ + MPID_nem_ib_vc_area *vc_ib = VC_IB(req->ch.vc); + vc_ib->ibcom->ncom_lmt_put -= 1; + MPID_nem_ib_ncqe_lmt_put -= 1; + dprintf("drain_scq_lmt_put,rndv,ncqe=%d\n", MPID_nem_ib_ncqe_lmt_put); /*suspicious */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); (VC_FIELD(req->ch.vc, pending_sends)) -= 1; @@ -491,7 +491,7 @@ int MPID_nem_dcfa_drain_scq_lmt_put() dprintf("drain_scq_lmt_put,unknown kind=%d,req_type=%d,msg_type=%d\n", kind, req_type, msg_type); #if 1 // lazy consulting of completion queue - MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_drain_scq_lmt_put"); + MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq_lmt_put"); #else //printf("kind=%d\n", kind); #endif @@ -499,33 +499,33 @@ int MPID_nem_dcfa_drain_scq_lmt_put() } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_LMT_PUT); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ_LMT_PUT); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_drain_scq_scratch_pad +#define FUNCNAME MPID_nem_ib_drain_scq_scratch_pad #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_drain_scq_scratch_pad() +int MPID_nem_ib_drain_scq_scratch_pad() { int mpi_errno = MPI_SUCCESS; int result; int i; - struct ibv_wc cqe[IBCOM_MAX_CQ_HEIGHT_DRAIN]; + struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_SCRATCH_PAD); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_SCRATCH_PAD); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ_SCRATCH_PAD); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ_SCRATCH_PAD); #if 0 /*def HAVE_LIBDCFA */ - result = ibv_poll_cq(rc_shared_scq_scratch_pad, 1, &cqe[0]); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, 1, &cqe[0]); #else - result = ibv_poll_cq(rc_shared_scq_scratch_pad, IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_scratch_pad, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); #endif - MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,dcfa,ibv_poll_cq"); + MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { dprintf("drain_scq_scratch_pad,found,result=%d\n", result); @@ -543,89 +543,89 @@ int MPID_nem_dcfa_drain_scq_scratch_pad() } #endif MPIU_ERR_CHKANDJUMP(cqe[i].status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq_scratch_pad"); + "**MPID_nem_ib_drain_scq_scratch_pad"); - IbCom *ibcom_scratch_pad = (IbCom *) cqe[i].wr_id; + MPID_nem_ib_com_t *ibcom_scratch_pad = (MPID_nem_ib_com_t *) cqe[i].wr_id; dprintf("drain_scq_scratch_pad,ibcom_scratch_pad=%p\n", ibcom_scratch_pad); ibcom_scratch_pad->ncom_scratch_pad -= 1; - MPID_nem_dcfa_ncqe_scratch_pad -= 1; + MPID_nem_ib_ncqe_scratch_pad -= 1; } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_DRAIN_SCQ_SCRATCH_PAD); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_DRAIN_SCQ_SCRATCH_PAD); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_poll_eager +#define FUNCNAME MPID_nem_ib_poll_eager #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) +int MPID_nem_ib_poll_eager(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; int result; - struct ibv_wc cqe[IBCOM_MAX_CQ_HEIGHT_DRAIN]; + struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; uint64_t tscs, tsce; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_POLL_EAGER); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_POLL_EAGER); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_POLL_EAGER); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_POLL_EAGER); - //MPID_nem_dcfa_tsc_poll = MPID_nem_dcfa_rdtsc(); + //MPID_nem_ib_tsc_poll = MPID_nem_ib_rdtsc(); - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - //dprintf("dcfa_poll,ld,rsr_seq_num_poll=%d\n", vc_dcfa->ibcom->rsr_seq_num_poll); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + //dprintf("ib_poll,ld,rsr_seq_num_poll=%d\n", vc_ib->ibcom->rsr_seq_num_poll); volatile void *buf = - (uint8_t *) vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + - IBCOM_RDMABUF_SZSEG * ((uint32_t) vc_dcfa->ibcom->rsr_seq_num_poll % IBCOM_RDMABUF_NSEG); - volatile sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf; - if (sz_hdrmagic->magic != IBCOM_MAGIC) { + (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * ((uint32_t) vc_ib->ibcom->rsr_seq_num_poll % MPID_NEM_IB_COM_RDMABUF_NSEG); + volatile MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf; + if (sz_hdrmagic->magic != MPID_NEM_IB_COM_MAGIC) { goto fn_exit; } - //dprintf("dcfa_poll_eager,buf=%p,sz=%d\n", buf, sz_hdrmagic->sz); + //dprintf("ib_poll_eager,buf=%p,sz=%d\n", buf, sz_hdrmagic->sz); /* unmark magic */ sz_hdrmagic->magic = 0 /*0xdead */ ; #if 0 - ibcom_errno = ibcom_poll_cq(IBCOM_RC_SHARED_RCQ, &cqe, &result); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_poll_cq"); + ibcom_errno = MPID_nem_ib_com_poll_cq(MPID_NEM_IB_COM_RC_SHARED_RCQ, &cqe, &result); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_poll_cq"); #endif - dprintf("dcfa_poll_eager,eager-send,found\n"); + dprintf("ib_poll_eager,eager-send,found\n"); - //MPIU_ERR_CHKANDJUMP1(cqe.status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**ibcom_poll_cq", "**ibcom_poll_cq %s", ibcom_strerror(ibcom_errno)); + //MPIU_ERR_CHKANDJUMP1(cqe.status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_poll_cq", "**MPID_nem_ib_com_poll_cq %s", MPID_nem_ib_com_strerror(ibcom_errno)); int sz_data_pow2; - DCFA_NEM_SZ_DATA_POW2(sz_hdrmagic->sz); - volatile tailmagic_t *tailmagic = (tailmagic_t *) ((uint8_t *) buf + sz_data_pow2); + MPID_NEM_IB_SZ_DATA_POW2(sz_hdrmagic->sz); + volatile MPID_nem_ib_tailmagic_t *tailmagic = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + sz_data_pow2); dprintf("poll,sz_data_pow2=%d,tailmagic=%p,sz=%d\n", sz_data_pow2, tailmagic, sz_hdrmagic->sz); int k = 0; - //tsce = MPID_nem_dcfa_rdtsc(); printf("9,%ld\n", tsce - tscs); // 55 for 512-byte - //tscs = MPID_nem_dcfa_rdtsc(); - //#define TLB_PREF_AMT_AHEAD 20 -#ifdef TLB_PREF_AMT_AHEAD - int tlb_pref_ahd = (uint64_t) tailmagic + 4096 * TLB_PREF_AMT_AHEAD - (uint64_t) buf; + //tsce = MPID_nem_ib_rdtsc(); printf("9,%ld\n", tsce - tscs); // 55 for 512-byte + //tscs = MPID_nem_ib_rdtsc(); + //#define MPID_NEM_IB_TLBPREF_POLL 20 +#ifdef MPID_NEM_IB_TLBPREF_POLL + int tlb_pref_ahd = (uint64_t) tailmagic + 4096 * MPID_NEM_IB_TLBPREF_POLL - (uint64_t) buf; #endif - while (tailmagic->magic != IBCOM_MAGIC) { + while (tailmagic->magic != MPID_NEM_IB_COM_MAGIC) { //k++; #if 0 /* pre-fetch next RDMA-write-buf slot to cover TLB miss latency */ __asm__ __volatile__ ("movq %0, %%rsi;" - "movq 0(%%rsi), %%rsi;"::"r"(vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + - IBCOM_RDMABUF_SZSEG * - ((vc_dcfa->ibcom->rsr_seq_num_poll + - 1) % IBCOM_RDMABUF_NSEG)):"%rsi"); + "movq 0(%%rsi), %%rsi;"::"r"(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + + MPID_NEM_IB_COM_RDMABUF_SZSEG * + ((vc_ib->ibcom->rsr_seq_num_poll + + 1) % MPID_NEM_IB_COM_RDMABUF_NSEG)):"%rsi"); #endif -#ifdef TLB_PREF_AMT_AHEAD +#ifdef MPID_NEM_IB_TLBPREF_POLL __asm__ __volatile__ ("movq %0, %%rsi;" "movq 0(%%rsi), %%rax;"::"r"(buf + tlb_pref_ahd):"%rsi", "%rax"); - tlb_pref_ahd = (tlb_pref_ahd + 4096 * 20) % IBCOM_RDMABUF_SZ; + tlb_pref_ahd = (tlb_pref_ahd + 4096 * 20) % MPID_NEM_IB_COM_RDMABUF_SZ; #endif } - //tsce = MPID_nem_dcfa_rdtsc(); printf("0,%ld\n", tsce - tscs); // 20-60 for 512-byte - //tscs = MPID_nem_dcfa_rdtsc(); + //tsce = MPID_nem_ib_rdtsc(); printf("0,%ld\n", tsce - tscs); // 20-60 for 512-byte + //tscs = MPID_nem_ib_rdtsc(); //dprintf("magic wait=%d\n", k); @@ -656,53 +656,53 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) } #endif - MPIDI_CH3_Pkt_eager_send_t *pkt = (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t)); + MPIDI_CH3_Pkt_eager_send_t *pkt = (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t)); MPIU_Assert(sz_hdrmagic->sz >= - sizeof(sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(tailmagic_t)); + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_tailmagic_t)); MPIDI_CH3_Pkt_eager_send_t *pkt2 = - (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t) + - sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t) + + sizeof(MPID_nem_ib_pkt_prefix_t)); dprintf ("handle_pkt,before,%d<-%d,id=%d,pkt->type=%d,pcc=%d,MPIDI_NEM_PKT_END=%d,pkt=%p,subtype=%d\n", - MPID_nem_dcfa_myrank, vc->pg_rank, vc_dcfa->ibcom->rsr_seq_num_poll, pkt->type, + MPID_nem_ib_myrank, vc->pg_rank, vc_ib->ibcom->rsr_seq_num_poll, pkt->type, MPIDI_CH3I_progress_completion_count.v, MPIDI_NEM_PKT_END, pkt, ((MPID_nem_pkt_netmod_t *) pkt)->subtype); /* see MPIDI_CH3_PktHandler_EagerSend (in src/mpid/ch3/src/ch3u_eager.c) */ mpi_errno = - MPID_nem_handle_pkt(vc, (char *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t)), - (MPIDI_msg_sz_t) (sz_hdrmagic->sz - sizeof(sz_hdrmagic_t) - - sizeof(tailmagic_t))); + MPID_nem_handle_pkt(vc, (char *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t)), + (MPIDI_msg_sz_t) (sz_hdrmagic->sz - sizeof(MPID_nem_ib_sz_hdrmagic_t) - + sizeof(MPID_nem_ib_tailmagic_t))); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - //tsce = MPID_nem_dcfa_rdtsc(); printf("0,%ld\n", tsce - tscs); // 512-byte, 900 cyc (1100 w/o prefetch) + //tsce = MPID_nem_ib_rdtsc(); printf("0,%ld\n", tsce - tscs); // 512-byte, 900 cyc (1100 w/o prefetch) /* Update occupation status of remote SR (send request) queue */ /* this includes local RDMA-wr-to buf occupation * because MPID_nem_handle_pkt releases RDMA-wr-to buf by copying data out */ /* responder releases resource and then embed largest sequence number into MPI message bound to initiator */ - //dprintf("after handle_pkt,rsr_seq_num_tail=%d\n", vc_dcfa->ibcom->rsr_seq_num_tail); + //dprintf("after handle_pkt,rsr_seq_num_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail); #if 1 dprintf ("handle_pkt,after,%d<-%d,id=%d,pkt->type=%d,eagershort=%d,close=%d,rts=%d,piggy-backed-eagersend=%d\n", - MPID_nem_dcfa_myrank, vc->pg_rank, vc_dcfa->ibcom->rsr_seq_num_poll, pkt->type, + MPID_nem_ib_myrank, vc->pg_rank, vc_ib->ibcom->rsr_seq_num_poll, pkt->type, MPIDI_CH3_PKT_EAGERSHORT_SEND, MPIDI_CH3_PKT_CLOSE, MPIDI_NEM_PKT_LMT_RTS, - MPIDI_NEM_DCFA_PKT_EAGER_SEND); + MPIDI_NEM_IB_PKT_EAGER_SEND); int notify_rate; ibcom_errno = - ibcom_rdmabuf_occupancy_notify_rate_get(MPID_nem_dcfa_conns[vc->pg_rank].fd, &notify_rate); + MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, &notify_rate); dprintf("poll_eager,sendq=%d,ncom=%d,ncqe=%d,ldiff=%d(%d-%d),rdiff=%d(%d-%d),rate=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail), - vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - vc_dcfa->ibcom->rsr_seq_num_tail, vc_dcfa->ibcom->rsr_seq_num_tail_last_sent, + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), + vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent, notify_rate); - //dprintf("dcfa_poll,current pcc=%d\n", MPIDI_CH3I_progress_completion_count.v); + //dprintf("ib_poll,current pcc=%d\n", MPIDI_CH3I_progress_completion_count.v); /* Don't forget to put lmt-cookie types here!! */ if (1) { @@ -712,21 +712,21 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) * because there is no way to trace the RDMA-write-to buffer addr * because rreq->dev.tmpbuf is set to zero in ch3_eager.c */ - dprintf("poll_eager,released,type=%d,MPIDI_NEM_DCFA_REPLY_SEQ_NUM=%d\n", pkt->type, - MPIDI_NEM_DCFA_REPLY_SEQ_NUM); - MPID_nem_dcfa_recv_buf_released(vc, - (void *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t) + + dprintf("poll_eager,released,type=%d,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM=%d\n", pkt->type, + MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM); + MPID_nem_ib_recv_buf_released(vc, + (void *) ((uint8_t *) buf + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t))); } else { if (sz_hdrmagic->sz == - sizeof(sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(tailmagic_t)) { + sizeof(MPID_nem_ib_sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(MPID_nem_ib_tailmagic_t)) { if (pkt->type == MPIDI_CH3_PKT_EAGERSHORT_SEND //|| pkt->type == MPIDI_CH3_PKT_GET ) { } else { - printf("dcfa_poll,unknown pkt->type=%d\n", pkt->type); + printf("ib_poll,unknown pkt->type=%d\n", pkt->type); assert(0); MPIU_ERR_INTERNALANDJUMP(mpi_errno, "MPI header only but not released"); } @@ -734,29 +734,29 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) } #endif - vc_dcfa->ibcom->rsr_seq_num_poll += 1; - dprintf("dcfa_poll,inc,rsr_seq_num_poll=%d\n", vc_dcfa->ibcom->rsr_seq_num_poll); + vc_ib->ibcom->rsr_seq_num_poll += 1; + dprintf("ib_poll,inc,rsr_seq_num_poll=%d\n", vc_ib->ibcom->rsr_seq_num_poll); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_POLL_EAGER); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_POLL_EAGER); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_poll +#define FUNCNAME MPID_nem_ib_poll #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_poll(int in_blocking_poll) +int MPID_nem_ib_poll(int in_blocking_poll) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; uint32_t i; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_POLL); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_POLL); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_POLL); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_POLL); #if 1 unsigned int progress_completion_count_old = MPIDI_CH3I_progress_completion_count.v; @@ -771,20 +771,20 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) * you should perform lmt-poll first, next eager-poll */ MPID_Request *rreq, *prev_rreq; - rreq = MPID_nem_dcfa_lmtq_head(MPID_nem_dcfa_lmtq); + rreq = MPID_nem_ib_lmtq_head(MPID_nem_ib_lmtq); if (rreq) { -#if defined (TIMER_WAIT_DCFA_POLL) +#if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { tsc[0] = MPI_rdtsc(); } #endif - // dprintf("dcfa_poll,poll lmtq\n"); + // dprintf("ib_poll,poll lmtq\n"); prev_rreq = NULL; do { /* Obtain cookie. pkt_RTS_handler memcpy it (in mpid_nem_lmt.c) */ /* MPID_IOV_BUF is macro, converted into iov_base (in src/include/mpiiov.h) */ /* do not use s_cookie_buf because do_cts frees it */ - //MPID_nem_dcfa_lmt_cookie_t* s_cookie_buf = (MPID_nem_dcfa_lmt_cookie_t*)rreq->ch.lmt_tmp_cookie.iov_base; + //MPID_nem_ib_lmt_cookie_t* s_cookie_buf = (MPID_nem_ib_lmt_cookie_t*)rreq->ch.lmt_tmp_cookie.iov_base; /* Wait for completion of DMA */ /* do not use s_cookie_buf->sz because do_cts frees it */ @@ -808,17 +808,17 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) if (*tailmagic != REQ_FIELD(rreq, lmt_tail)) { goto next; } - dprintf("dcfa_poll,sz=%ld,old tail=%02x,new tail=%02x\n", rreq->ch.lmt_data_sz, + dprintf("ib_poll,sz=%ld,old tail=%02x,new tail=%02x\n", rreq->ch.lmt_data_sz, REQ_FIELD(rreq, lmt_tail), *tailmagic); dprintf - ("dcfa_poll,lmt found,%d<-%d,req=%p,ref_count=%d,is_contig=%d,write_to_buf=%p,lmt_pack_buf=%p,user_buf=%p,tail=%p\n", - MPID_nem_dcfa_myrank, rreq->ch.vc->pg_rank, rreq, rreq->ref_count, is_contig, + ("ib_poll,lmt found,%d<-%d,req=%p,ref_count=%d,is_contig=%d,write_to_buf=%p,lmt_pack_buf=%p,user_buf=%p,tail=%p\n", + MPID_nem_ib_myrank, rreq->ch.vc->pg_rank, rreq, rreq->ref_count, is_contig, write_to_buf, REQ_FIELD(rreq, lmt_pack_buf), rreq->dev.user_buf, tailmagic); /* unpack non-contiguous dt */ if (!is_contig) { - dprintf("dcfa_poll,copying noncontiguous data to user buffer\n"); + dprintf("ib_poll,copying noncontiguous data to user buffer\n"); /* see MPIDI_CH3U_Request_unpack_uebuf (in /src/mpid/ch3/src/ch3u_request.c) */ /* or MPIDI_CH3U_Receive_data_found (in src/mpid/ch3/src/ch3u_handle_recv_pkt.c) */ @@ -838,46 +838,46 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) MPIR_STATUS_SET_COUNT(rreq->status, last); rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, - MPI_ERR_TYPE, "**MPID_nem_dcfa_poll", 0); + MPI_ERR_TYPE, "**MPID_nem_ib_poll", 0); /* --END ERROR HANDLING-- */ } #if 1 /* debug, enable again later, polling waits forever when freeing it here. */ //if (rreq->ref_count == 1) { - dprintf("dcfa_poll,lmt,ref_count=%d,lmt_pack_buf=%p\n", rreq->ref_count, + dprintf("ib_poll,lmt,ref_count=%d,lmt_pack_buf=%p\n", rreq->ref_count, REQ_FIELD(rreq, lmt_pack_buf)); //MPIU_Free(REQ_FIELD(rreq, lmt_pack_buf)); - MPID_nem_dcfa_stfree(REQ_FIELD(rreq, lmt_pack_buf), (size_t) rreq->ch.lmt_data_sz); + MPID_nem_ib_stfree(REQ_FIELD(rreq, lmt_pack_buf), (size_t) rreq->ch.lmt_data_sz); //} else { - // dprintf("dcfa_poll,lmt,ref_count=%d,lmt_pack_buf=%p\n", rreq->ref_count, REQ_FIELD(rreq, lmt_pack_buf)); + // dprintf("ib_poll,lmt,ref_count=%d,lmt_pack_buf=%p\n", rreq->ref_count, REQ_FIELD(rreq, lmt_pack_buf)); //} #endif } - /* send done to sender. vc is stashed in MPID_nem_dcfa_lmt_start_recv (in dcfa_lmt.c) */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(rreq->ch.vc); - dprintf("dcfa_poll,GET,lmt_send_GET_DONE,rsr_seq_num_tail=%d\n", - vc_dcfa->ibcom->rsr_seq_num_tail); - MPID_nem_dcfa_lmt_send_GET_DONE(rreq->ch.vc, rreq); - dprintf("dcfa_poll,prev_rreq=%p,rreq->lmt_next=%p\n", prev_rreq, - MPID_nem_dcfa_lmtq_next(rreq)); + /* send done to sender. vc is stashed in MPID_nem_ib_lmt_start_recv (in ib_lmt.c) */ + MPID_nem_ib_vc_area *vc_ib = VC_IB(rreq->ch.vc); + dprintf("ib_poll,GET,lmt_send_GET_DONE,rsr_seq_num_tail=%d\n", + vc_ib->ibcom->rsr_seq_num_tail); + MPID_nem_ib_lmt_send_GET_DONE(rreq->ch.vc, rreq); + dprintf("ib_poll,prev_rreq=%p,rreq->lmt_next=%p\n", prev_rreq, + MPID_nem_ib_lmtq_next(rreq)); /* unlink rreq */ if (prev_rreq != NULL) { - MPID_nem_dcfa_lmtq_next(prev_rreq) = MPID_nem_dcfa_lmtq_next(rreq); + MPID_nem_ib_lmtq_next(prev_rreq) = MPID_nem_ib_lmtq_next(rreq); } else { - MPID_nem_dcfa_lmtq_head(MPID_nem_dcfa_lmtq) = MPID_nem_dcfa_lmtq_next(rreq); + MPID_nem_ib_lmtq_head(MPID_nem_ib_lmtq) = MPID_nem_ib_lmtq_next(rreq); } - if (MPID_nem_dcfa_lmtq_next(rreq) == NULL) { - MPID_nem_dcfa_lmtq.tail = prev_rreq; + if (MPID_nem_ib_lmtq_next(rreq) == NULL) { + MPID_nem_ib_lmtq.tail = prev_rreq; } /* save rreq->dev.next (and rreq) because decrementing reference-counter might free rreq */ MPID_Request *tmp_rreq = rreq; - rreq = MPID_nem_dcfa_lmtq_next(rreq); + rreq = MPID_nem_ib_lmtq_next(rreq); /* decrement completion-counter */ - dprintf("dcfa_poll,%d<-%d,", MPID_nem_dcfa_myrank, tmp_rreq->ch.vc->pg_rank); + dprintf("ib_poll,%d<-%d,", MPID_nem_ib_myrank, tmp_rreq->ch.vc->pg_rank); int incomplete; MPIDI_CH3U_Request_decrement_cc(tmp_rreq, &incomplete); dprintf("lmt,complete,tmp_rreq=%p,rreq->ref_count=%d,comm=%p\n", tmp_rreq, @@ -888,82 +888,82 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) } /* lmt_start_recv increments ref_count - * drain_scq and dcfa_poll is not ordered, so both can decrement ref_count */ + * drain_scq and ib_poll is not ordered, so both can decrement ref_count */ /* ref_count is decremented - * get-lmt: dcfa_poll, drain_scq, wait - * put-lmt: dcfa_poll, wait */ + * get-lmt: ib_poll, drain_scq, wait + * put-lmt: ib_poll, wait */ MPID_Request_release(tmp_rreq); - dprintf("dcfa_poll,lmt,after release,tmp_rreq=%p,rreq->ref_count=%d,comm=%p\n", + dprintf("ib_poll,lmt,after release,tmp_rreq=%p,rreq->ref_count=%d,comm=%p\n", tmp_rreq, tmp_rreq->ref_count, tmp_rreq->comm); goto next_unlinked; next: prev_rreq = rreq; - rreq = MPID_nem_dcfa_lmtq_next(rreq); + rreq = MPID_nem_ib_lmtq_next(rreq); next_unlinked:; } while (rreq); -#if defined (TIMER_WAIT_DCFA_POLL) +#if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { stsc[0] += MPI_rdtsc() - tsc[0]; } #endif } -#if defined (TIMER_WAIT_DCFA_POLL) +#if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { tsc[1] = MPI_rdtsc(); } #endif int ncom_almost_full = 0; - for (i = 0; i < MPID_nem_dcfa_npollingset; i++) { - //tscs = MPID_nem_dcfa_rdtsc(); - MPIDI_VC_t *vc = MPID_nem_dcfa_pollingset[i]; - mpi_errno = MPID_nem_dcfa_poll_eager(vc); + for (i = 0; i < MPID_nem_ib_npollingset; i++) { + //tscs = MPID_nem_ib_rdtsc(); + MPIDI_VC_t *vc = MPID_nem_ib_pollingset[i]; + mpi_errno = MPID_nem_ib_poll_eager(vc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); /* without this, command in sendq doesn't have a chance * to perform send_progress * when send and progress_send call drain_scq but asking it * for not performing send_progress and make the CQ empty */ - MPID_nem_dcfa_send_progress(vc_dcfa); + MPID_nem_ib_send_progress(vc_ib); - ncom_almost_full |= (vc_dcfa->ibcom->ncom >= IBCOM_MAX_SQ_HEIGHT_DRAIN); + ncom_almost_full |= (vc_ib->ibcom->ncom >= MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN); #if 0 /* aggressively perform drain_scq */ - ncom_almost_full |= !(MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)); + ncom_almost_full |= !(MPID_nem_ib_sendq_empty(vc_ib->sendq)); #endif } -#if defined (TIMER_WAIT_DCFA_POLL) +#if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { stsc[1] += MPI_rdtsc() - tsc[1]; } #endif // lazy feching of completion queue entry because it causes cache-miss -#if defined (LMT_GET_CQE) - if (MPID_nem_dcfa_ncqe_to_drain > 0 || MPID_nem_dcfa_ncqe_nces > 0 || - MPID_nem_dcfa_ncqe >= IBCOM_MAX_CQ_HEIGHT_DRAIN || ncom_almost_full) +#if defined (MPID_NEM_IB_LMT_GET_CQE) + if (MPID_nem_ib_ncqe_to_drain > 0 || MPID_nem_ib_ncqe_nces > 0 || + MPID_nem_ib_ncqe >= MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN || ncom_almost_full) #endif -#if !defined (LMT_GET_CQE) - if (/*(in_blocking_poll && result == 0) || */ MPID_nem_dcfa_ncqe_nces > 0 || - MPID_nem_dcfa_ncqe >= IBCOM_MAX_CQ_HEIGHT_DRAIN || ncom_almost_full) +#if !defined (MPID_NEM_IB_LMT_GET_CQE) + if (/*(in_blocking_poll && result == 0) || */ MPID_nem_ib_ncqe_nces > 0 || + MPID_nem_ib_ncqe >= MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN || ncom_almost_full) #endif { -#if defined (TIMER_WAIT_DCFA_POLL) +#if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { tsc[0] = MPI_rdtsc(); } #endif - //dprintf("dcfa_poll,calling drain_scq\n"); - ibcom_errno = MPID_nem_dcfa_drain_scq(0); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_drain_scq"); -#if defined (TIMER_WAIT_DCFA_POLL) + //dprintf("ib_poll,calling drain_scq\n"); + ibcom_errno = MPID_nem_ib_drain_scq(0); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); +#if defined (MPID_NEM_IB_TIMER_WAIT_IB_POLL) if (in_blocking_poll) { stsc[0] += MPI_rdtsc() - tsc[0]; } @@ -971,24 +971,24 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) } #if 1 /* aggressively perform drain_scq */ - ibcom_errno = MPID_nem_dcfa_drain_scq(0); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_drain_scq"); + ibcom_errno = MPID_nem_ib_drain_scq(0); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); #endif - /* detect completion of lmt-put when MPI_Wait kicks dcfa_poll */ - if (MPID_nem_dcfa_ncqe_lmt_put > 0) { - ibcom_errno = MPID_nem_dcfa_drain_scq_lmt_put(); + /* detect completion of lmt-put when MPI_Wait kicks ib_poll */ + if (MPID_nem_ib_ncqe_lmt_put > 0) { + ibcom_errno = MPID_nem_ib_drain_scq_lmt_put(); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq_lmt_put"); + "**MPID_nem_ib_drain_scq_lmt_put"); } -#ifdef DCFA_ONDEMAND +#ifdef MPID_NEM_IB_ONDEMAND /* process incoming connection request */ - MPID_nem_dcfa_cm_accept(); + MPID_nem_ib_cm_accept(); /* process outgoing conncetion request */ - if (MPID_nem_dcfa_ncqe_connect >= IBCOM_MAX_CQ_HEIGHT_DRAIN) { - ibcom_errno = MPID_nem_dcfa_cm_drain_scq(0); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_cm_drain_scq"); + if (MPID_nem_ib_ncqe_connect >= MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { + ibcom_errno = MPID_nem_ib_cm_drain_scq(0); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_drain_scq"); } #endif @@ -1001,7 +1001,7 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) //if (in_blocking_poll) { goto prev; } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_POLL); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_POLL); return mpi_errno; fn_fail: goto fn_exit; @@ -1012,23 +1012,23 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) * so that we can obtain rreq by ibv_poll_cq */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_recv_posted +#define FUNCNAME MPID_nem_ib_recv_posted #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) +int MPID_nem_ib_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) { int mpi_errno = MPI_SUCCESS; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_RECV_POSTED); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_RECV_POSTED); - dprintf("recv_posted,enter,%d->%d,req=%p\n", MPID_nem_dcfa_myrank, vc->pg_rank, req); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_RECV_POSTED); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_RECV_POSTED); + dprintf("recv_posted,enter,%d->%d,req=%p\n", MPID_nem_ib_myrank, vc->pg_rank, req); #if 0 int ibcom_errno; - ibcom_errno = ibcom_irecv(vc_dcfa->sc->fd, (uint64_t) vc->pg_rank); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_irecv"); + ibcom_errno = MPID_nem_ib_com_irecv(vc_ib->sc->fd, (uint64_t) vc->pg_rank); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_irecv"); #endif #if 1 /*takagi */ @@ -1042,11 +1042,11 @@ int MPID_nem_dcfa_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) /* poll when rreq is for lmt */ /* anticipating received message finds maching request in the posted-queue */ if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) > vc->eager_max_msg_sz) { - //if (MPID_nem_dcfa_tsc_poll - MPID_nem_dcfa_rdtsc() > MPID_NEM_DCFA_POLL_PERIOD_RECV_POSTED) { + //if (MPID_nem_ib_tsc_poll - MPID_nem_ib_rdtsc() > MPID_NEM_IB_POLL_PERIOD_RECV_POSTED) { #if 1 - mpi_errno = MPID_nem_dcfa_poll_eager(vc); + mpi_errno = MPID_nem_ib_poll_eager(vc); #else - mpi_errno = MPID_nem_dcfa_poll(0); + mpi_errno = MPID_nem_ib_poll(0); #endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -1056,11 +1056,11 @@ int MPID_nem_dcfa_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) else { #if 1 /* anticipating received message finds maching request in the posted-queue */ - //if (MPID_nem_dcfa_tsc_poll - MPID_nem_dcfa_rdtsc() > MPID_NEM_DCFA_POLL_PERIOD_RECV_POSTED) { + //if (MPID_nem_ib_tsc_poll - MPID_nem_ib_rdtsc() > MPID_NEM_IB_POLL_PERIOD_RECV_POSTED) { #if 1 - mpi_errno = MPID_nem_dcfa_poll_eager(vc); + mpi_errno = MPID_nem_ib_poll_eager(vc); #else - mpi_errno = MPID_nem_dcfa_poll(0); + mpi_errno = MPID_nem_ib_poll(0); #endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); @@ -1071,7 +1071,7 @@ int MPID_nem_dcfa_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) #endif fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_RECV_POSTED); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_RECV_POSTED); return mpi_errno; fn_fail: goto fn_exit; @@ -1085,22 +1085,22 @@ int MPID_nem_dcfa_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req) suffices in original MPICH implementation) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_recv_buf_released +#define FUNCNAME MPID_nem_ib_recv_buf_released #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) +int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_RECV_BUF_RELEASED); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_RECV_BUF_RELEASED); - dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_dcfa_myrank, vc->pg_rank, + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); + dprintf("recv_buf_released,%d<-%d,user_data=%p\n", MPID_nem_ib_myrank, vc->pg_rank, user_data); -#if 1 /* moving from dcfa_poll */ +#if 1 /* moving from ib_poll */ /* unmark magic */ - /* magic is located at IBCOM_INLINE_DATA boundary and variable length entails multiple prospective locations for the future use */ + /* magic is located at MPID_NEM_IB_COM_INLINE_DATA boundary and variable length entails multiple prospective locations for the future use */ /* see MPIDI_CH3_PktHandler_EagerShortSend (in src/mpid/ch3/src/ch3u_eager.c */ /* eager-send with zero-length data is released in poll @@ -1111,29 +1111,29 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) goto fn_exit; } - MPIU_Assert(vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] <= user_data && - user_data < vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZ); + MPIU_Assert(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] <= user_data && + user_data < vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ); unsigned long mod = (unsigned long) ((uint8_t *) user_data - - (uint8_t *) vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO]) & (IBCOM_RDMABUF_SZSEG - 1); + (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) & (MPID_NEM_IB_COM_RDMABUF_SZSEG - 1); void *buf = (void *) ((uint8_t *) user_data - mod); //dprintf("recv_buf_released,clearing,buf=%p\n", buf); - sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf; + MPID_nem_ib_sz_hdrmagic_t *sz_hdrmagic = (MPID_nem_ib_sz_hdrmagic_t *) buf; int sz_data_pow2; - DCFA_NEM_SZ_DATA_POW2(sz_hdrmagic->sz); + MPID_NEM_IB_SZ_DATA_POW2(sz_hdrmagic->sz); //dprintf("recv_buf_released,sz=%d,pow2=%d\n", sz_hdrmagic->sz, sz_data_pow2); #if 1 uint32_t offset; for (offset = 0;; offset = offset ? ((((offset + 1) << 1) - 1) > - DCFA_NEM_MAX_DATA_POW2 ? DCFA_NEM_MAX_DATA_POW2 : (((offset + 1) << 1) - + MPID_NEM_IB_MAX_DATA_POW2 ? MPID_NEM_IB_MAX_DATA_POW2 : (((offset + 1) << 1) - 1)) : 15) { - volatile tailmagic_t *ptr = (tailmagic_t *) ((uint8_t *) buf + offset); - MPIU_Assert(vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] <= ptr && - ptr < vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZ); + volatile MPID_nem_ib_tailmagic_t *ptr = (MPID_nem_ib_tailmagic_t *) ((uint8_t *) buf + offset); + MPIU_Assert(vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] <= ptr && + ptr < vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO] + MPID_NEM_IB_COM_RDMABUF_SZ); ptr->magic = 0 /*0xde */ ; if (offset == sz_data_pow2) { break; @@ -1142,28 +1142,28 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) #endif #endif -#if 1 /* moving from dcfa_poll */ +#if 1 /* moving from ib_poll */ /* mark that one eager-send RDMA-write-to buffer has been released */ int index_slot = (unsigned long) ((uint8_t *) user_data - - (uint8_t *) vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO]) / IBCOM_RDMABUF_SZSEG; - MPIU_Assert(0 <= index_slot && index_slot < IBCOM_RDMABUF_NSEG); - //dprintf("user_data=%p,mem=%p,sub=%08lx,index_slot=%d\n", user_data, vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO], (unsigned long)user_data - (unsigned long)vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO], index_slot); - //dprintf("index_slot=%d,released=%016lx\n", index_slot, vc_dcfa->ibcom->rsr_seq_num_released[index_slot / 64]); - vc_dcfa->ibcom->rsr_seq_num_released[index_slot / 64] |= (1ULL << (index_slot & 63)); - //dprintf("released[index_slot/64]=%016lx\n", vc_dcfa->ibcom->rsr_seq_num_released[index_slot / 64]); - // int index_tail = (vc_dcfa->ibcom->rsr_seq_num_tail + 1) & (IBCOM_RDMABUF_NSEG-1); - int index_tail = (vc_dcfa->ibcom->rsr_seq_num_tail + 1) % IBCOM_RDMABUF_NSEG; - //dprintf("tail+1=%d,index_tail=%d\n", vc_dcfa->ibcom->rsr_seq_num_tail + 1, index_tail); - //dprintf("released=%016lx\n", vc_dcfa->ibcom->rsr_seq_num_released[index_tail / 64]); - if (1 || (index_tail & 7) || MPID_nem_dcfa_diff32(index_slot, index_tail) >= IBCOM_RDMABUF_NSEG - 8) { /* avoid wrap-around */ + (uint8_t *) vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO]) / MPID_NEM_IB_COM_RDMABUF_SZSEG; + MPIU_Assert(0 <= index_slot && index_slot < MPID_NEM_IB_COM_RDMABUF_NSEG); + //dprintf("user_data=%p,mem=%p,sub=%08lx,index_slot=%d\n", user_data, vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], (unsigned long)user_data - (unsigned long)vc_ib->ibcom->icom_mem[MPID_NEM_IB_COM_RDMAWR_TO], index_slot); + //dprintf("index_slot=%d,released=%016lx\n", index_slot, vc_ib->ibcom->rsr_seq_num_released[index_slot / 64]); + vc_ib->ibcom->rsr_seq_num_released[index_slot / 64] |= (1ULL << (index_slot & 63)); + //dprintf("released[index_slot/64]=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_slot / 64]); + // int index_tail = (vc_ib->ibcom->rsr_seq_num_tail + 1) & (MPID_NEM_IB_COM_RDMABUF_NSEG-1); + int index_tail = (vc_ib->ibcom->rsr_seq_num_tail + 1) % MPID_NEM_IB_COM_RDMABUF_NSEG; + //dprintf("tail+1=%d,index_tail=%d\n", vc_ib->ibcom->rsr_seq_num_tail + 1, index_tail); + //dprintf("released=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_tail / 64]); + if (1 || (index_tail & 7) || MPID_nem_ib_diff32(index_slot, index_tail) >= MPID_NEM_IB_COM_RDMABUF_NSEG - 8) { /* avoid wrap-around */ while (1) { - if (((vc_dcfa->ibcom-> + if (((vc_ib->ibcom-> rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) { - vc_dcfa->ibcom->rsr_seq_num_tail += 1; - vc_dcfa->ibcom->rsr_seq_num_released[index_tail / 64] &= + vc_ib->ibcom->rsr_seq_num_tail += 1; + vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= ~(1ULL << (index_tail & 63)); - dprintf("rsr_seq_num_tail,incremented to %d\n", vc_dcfa->ibcom->rsr_seq_num_tail); + dprintf("rsr_seq_num_tail,incremented to %d\n", vc_ib->ibcom->rsr_seq_num_tail); } else { break; @@ -1171,48 +1171,48 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) } } else { - if (((vc_dcfa->ibcom->rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 0xff) == + if (((vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] >> (index_tail & 63)) & 0xff) == 0xff) { - vc_dcfa->ibcom->rsr_seq_num_tail += 8; - vc_dcfa->ibcom->rsr_seq_num_released[index_tail / 64] &= + vc_ib->ibcom->rsr_seq_num_tail += 8; + vc_ib->ibcom->rsr_seq_num_released[index_tail / 64] &= ~(0xffULL << (index_tail & 63)); - //dprintf("released[index_tail/64]=%016lx\n", vc_dcfa->ibcom->rsr_seq_num_released[index_tail / 64]); + //dprintf("released[index_tail/64]=%016lx\n", vc_ib->ibcom->rsr_seq_num_released[index_tail / 64]); } } - //dprintf("recv_buf_released,%d->%d,rsr_seq_num_tail=%d,rsr_seq_num_tail_last_sent=%d\n", MPID_nem_dcfa_myrank, vc->pg_rank, vc_dcfa->ibcom->rsr_seq_num_tail, vc_dcfa->ibcom->rsr_seq_num_tail_last_sent); + //dprintf("recv_buf_released,%d->%d,rsr_seq_num_tail=%d,rsr_seq_num_tail_last_sent=%d\n", MPID_nem_ib_myrank, vc->pg_rank, vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent); int notify_rate; ibcom_errno = - ibcom_rdmabuf_occupancy_notify_rate_get(MPID_nem_dcfa_conns[vc->pg_rank].fd, &notify_rate); + MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(MPID_nem_ib_conns[vc->pg_rank].fd, &notify_rate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**ibcom_rdmabuf_occupancy_notify_rate_get"); + "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get"); /* if you missed the chance to make eager-send message piggy-back it */ - if (MPID_nem_dcfa_diff32 - (vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent) > - IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) - //|| MPID_nem_dcfa_diff32(lsr_seq_num_head, vc_dcfa->ibcom->lsr_seq_num_tail_last_sent) == IBCOM_RDMABUF_NSEG + if (MPID_nem_ib_diff32 + (vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent) > + MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) + //|| MPID_nem_ib_diff32(lsr_seq_num_head, vc_ib->ibcom->lsr_seq_num_tail_last_sent) == MPID_NEM_IB_COM_RDMABUF_NSEG ) { MPID_Request *sreq; - sreq = MPID_nem_dcfa_sendq_head(vc_dcfa->sendq); + sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); if (sreq) { int msg_type = MPIDI_Request_get_msg_type(sreq); MPIDI_CH3_Pkt_t *ch3_hdr = (MPIDI_CH3_Pkt_t *) sreq->dev.iov[0].MPID_IOV_BUF; if (msg_type == MPIDI_REQUEST_EAGER_MSG && /* guard for the following pointer dereference */ - ch3_hdr->type == MPIDI_NEM_DCFA_REPLY_SEQ_NUM) { + ch3_hdr->type == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { goto skip; } } - //printf("recv_buf_released,sending reply_seq_num,diff=%d,rate=%d,id=%d\n", MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), notify_rate + (notify_rate>>1), vc_dcfa->ibcom->sseq_num); - MPID_nem_dcfa_send_reply_seq_num(vc); + //printf("recv_buf_released,sending reply_seq_num,diff=%d,rate=%d,id=%d\n", MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent), notify_rate + (notify_rate>>1), vc_ib->ibcom->sseq_num); + MPID_nem_ib_send_reply_seq_num(vc); skip:; } #endif fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_RECV_BUF_RELEASED); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_RECV_BUF_RELEASED); return mpi_errno; fn_fail: goto fn_exit; @@ -1222,23 +1222,23 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) /* packet handler for wrapper packet of MPIDI_NEM_PKT_LMT_DONE */ /* see pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_lmt_done +#define FUNCNAME MPID_nem_ib_PktHandler_lmt_done #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_pkt_lmt_done_t *const done_pkt = (MPID_nem_dcfa_pkt_lmt_done_t *) pkt; + MPID_nem_ib_pkt_lmt_done_t *const done_pkt = (MPID_nem_ib_pkt_lmt_done_t *) pkt; MPID_Request *req; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_LMT_DONE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_LMT_DONE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_LMT_DONE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_LMT_DONE); /* Check the assumption on sizeof(MPIDI_CH3_Pkt_t). * It is utilized in pkt_DONE_handler (in src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt.c) - * that must be larger than sizeof(MPID_nem_dcfa_pkt_lmt_done_t) */ - if (sizeof(MPID_nem_dcfa_pkt_lmt_done_t) > sizeof(MPIDI_CH3_Pkt_t)) { + * that must be larger than sizeof(MPID_nem_ib_pkt_lmt_done_t) */ + if (sizeof(MPID_nem_ib_pkt_lmt_done_t) > sizeof(MPIDI_CH3_Pkt_t)) { MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_INTERN, "**sizeof(MPIDI_CH3_Pkt_t)"); } @@ -1259,7 +1259,7 @@ int MPID_nem_dcfa_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_LMT_DONE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_LMT_DONE); return mpi_errno; fn_fail: goto fn_exit; @@ -1269,16 +1269,16 @@ int MPID_nem_dcfa_PktHandler_lmt_done(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* packet handler for wrapper packet of MPIDI_CH3_PKT_EAGER_SEND */ /* see MPIDI_CH3_PktHandler_EagerSend (in src/mpid/ch3/src/ch3u_eager.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_EagerSend +#define FUNCNAME MPID_nem_ib_PktHandler_EagerSend #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */) { - MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; + MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_eager_send_t *ch3_pkt = - (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1287,73 +1287,73 @@ int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_EAGERSEND); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_EAGERSEND); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_EAGERSEND); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_EAGERSEND); - dprintf("dcfa_pkthandler_eagersend,tag=%d\n", ch3_pkt->match.parts.tag); + dprintf("ib_pkthandler_eagersend,tag=%d\n", ch3_pkt->match.parts.tag); /* Check the assumption on sizeof(MPIDI_CH3_Pkt_t). * It is utilized to point the payload location in MPIDI_CH3_PktHandler_EagerSend - * (src/mpid/ch3/src/ch3u_eager.c) that must be larger than sizeof(MPID_nem_dcfa_pkt_eager_send_t) */ - //if (sizeof(MPID_nem_dcfa_pkt_eager_send_t) > sizeof(MPIDI_CH3_Pkt_t)) { + * (src/mpid/ch3/src/ch3u_eager.c) that must be larger than sizeof(MPID_nem_ib_pkt_eager_send_t) */ + //if (sizeof(MPID_nem_ib_pkt_eager_send_t) > sizeof(MPIDI_CH3_Pkt_t)) { //MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_INTERN, "**sizeof(MPIDI_CH3_Pkt_t)"); //} /* Update occupation status of local SR (send request) queue */ int *lsr_seq_num_tail; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - ibcom_errno = ibcom_lsr_seq_num_tail_get(vc_dcfa->sc->fd, &lsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_lsr_seq_num_tail_get"); - dprintf("MPID_nem_dcfa_PktHandler_EagerSend,lsr_seq_num_tail=%d,netmod_pkt->seq_num_tail=%d\n", + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get"); + dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail=%d,netmod_pkt->seq_num_tail=%d\n", *lsr_seq_num_tail, netmod_pkt->seq_num_tail); - *lsr_seq_num_tail = DCFA_MAX(*lsr_seq_num_tail, netmod_pkt->seq_num_tail); - dprintf("MPID_nem_dcfa_PktHandler_EagerSend,lsr_seq_num_tail updated to %d\n", + *lsr_seq_num_tail = MPID_NEM_IB_MAX(*lsr_seq_num_tail, netmod_pkt->seq_num_tail); + dprintf("MPID_nem_ib_PktHandler_EagerSend,lsr_seq_num_tail updated to %d\n", *lsr_seq_num_tail); -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ dprintf("pkthandler,eagersend,old rstate=%d\n", - vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, lsr_seq_num_tail); + vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, lsr_seq_num_tail); dprintf("pkthandler,eagersend,new rstate=%d\n", - vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,eagersend,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ /* calling drain_scq from progress_send derpives of chance - * for dcfa_poll to drain sendq using ncqe + * for ib_poll to drain sendq using ncqe * however transfers events to * (not to reply_seq_num because it's regulated by the rate) - * fire on dcfa_poll using nces (e.g. MPI_Put) so we need to perform - * progress_send for all of VCs using nces in dcfa_poll. */ + * fire on ib_poll using nces (e.g. MPI_Put) so we need to perform + * progress_send for all of VCs using nces in ib_poll. */ dprintf("pkthandler,eagersend,send_progress\n"); fflush(stdout); - MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS + MPID_NEM_IB_CHECK_AND_SEND_PROGRESS /* fall back to the original handler */ /* we don't need to worry about the difference caused by embedding seq_num * because size of MPI-header of MPIDI_CH3_PKT_EAGER_SEND equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_dcfa_iSendContig + * see MPID_nem_ib_iSendContig */ //ch3_pkt->type = MPIDI_CH3_PKT_EAGER_SEND; #if 0 mpi_errno = MPID_nem_handle_pkt(vc, (char *) pkt_parent_class, *buflen); #else - dprintf("dcfa_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_dcfa_pkt_prefix_t); + dprintf("ib_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); + MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_EagerSend(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - dprintf("dcfa_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen); - *buflen = ch3_buflen + sizeof(MPID_nem_dcfa_pkt_prefix_t); - dprintf("dcfa_poll.c,after addition,buflen=%ld\n", *buflen); + dprintf("ib_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen); + *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); + dprintf("ib_poll.c,after addition,buflen=%ld\n", *buflen); #endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_EAGERSEND); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_EAGERSEND); return mpi_errno; fn_fail: goto fn_exit; @@ -1363,16 +1363,16 @@ int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* packet handler for wrapper packet of MPIDI_CH3_PKT_PUT */ /* see MPIDI_CH3_PktHandler_EagerSend (in src/mpid/ch3/src/ch3u_rma_sync.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_Put +#define FUNCNAME MPID_nem_ib_PktHandler_Put #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */) { - MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; + MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_put_t *ch3_pkt = - (MPIDI_CH3_Pkt_put_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_put_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1381,47 +1381,47 @@ int MPID_nem_dcfa_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_PUT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_PUT); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_PUT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_PUT); /* Update occupation status of local SR (send request) queue */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - dprintf("MPID_nem_dcfa_Pkthandler_Put,lsr_seq_num_tail=%d,put_pkt->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - dprintf("MPID_nem_dcfa_Pkthandler_Put,lsr_seq_num_tail updated to %d\n", - vc_dcfa->ibcom->lsr_seq_num_tail); - -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + dprintf("MPID_nem_ib_Pkthandler_Put,lsr_seq_num_tail=%d,put_pkt->seq_num_tail=%d\n", + vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + dprintf("MPID_nem_ib_Pkthandler_Put,lsr_seq_num_tail updated to %d\n", + vc_ib->ibcom->lsr_seq_num_tail); + +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,put,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, - &vc_dcfa->ibcom->lsr_seq_num_tail); - dprintf("pkthandler,put,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, + &vc_ib->ibcom->lsr_seq_num_tail); + dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ dprintf("pkthandler,put,send_progress\n"); fflush(stdout); - MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS; + MPID_NEM_IB_CHECK_AND_SEND_PROGRESS; /* fall back to the original handler */ /* we don't need to worry about the difference caused by embedding seq_num * because size of MPI-header of MPIDI_CH3_PKT_PUT equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_dcfa_iSendContig + * see MPID_nem_ib_iSendContig */ - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_dcfa_pkt_prefix_t); + MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_Put(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - *buflen = ch3_buflen + sizeof(MPID_nem_dcfa_pkt_prefix_t); + *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_PUT); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_PUT); return mpi_errno; fn_fail: goto fn_exit; @@ -1431,16 +1431,16 @@ int MPID_nem_dcfa_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* packet handler for wrapper packet of MPIDI_CH3_PKT_ACCUMULATE */ /* see MPIDI_CH3_PktHandler_Accumulate (in src/mpid/ch3/src/ch3u_rma_sync.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_Accumulate +#define FUNCNAME MPID_nem_ib_PktHandler_Accumulate #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */) { - MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; + MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_accum_t *ch3_pkt = - (MPIDI_CH3_Pkt_accum_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_accum_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1449,47 +1449,47 @@ int MPID_nem_dcfa_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_ACCUMULATE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_ACCUMULATE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_ACCUMULATE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_ACCUMULATE); /* Update occupation status of local SR (send request) queue */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - dprintf("MPID_nem_dcfa_Pkthandler_Accumulate,lsr_seq_num_tail=%d,accum_pkt->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - dprintf("MPID_nem_dcfa_Pkthandler_Accumulate,lsr_seq_num_tail updated to %d\n", - vc_dcfa->ibcom->lsr_seq_num_tail); - -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + dprintf("MPID_nem_ib_Pkthandler_Accumulate,lsr_seq_num_tail=%d,accum_pkt->seq_num_tail=%d\n", + vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + dprintf("MPID_nem_ib_Pkthandler_Accumulate,lsr_seq_num_tail updated to %d\n", + vc_ib->ibcom->lsr_seq_num_tail); + +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,put,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, - &vc_dcfa->ibcom->lsr_seq_num_tail); - dprintf("pkthandler,put,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, + &vc_ib->ibcom->lsr_seq_num_tail); + dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ dprintf("pkthandler,put,send_progress\n"); fflush(stdout); - MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS + MPID_NEM_IB_CHECK_AND_SEND_PROGRESS /* fall back to the original handler */ /* we don't need to worry about the difference caused by embedding seq_num * because size of MPI-header of MPIDI_CH3_PKT_PUT equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_dcfa_iSendContig + * see MPID_nem_ib_iSendContig */ - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_dcfa_pkt_prefix_t); + MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_Accumulate(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - *buflen = ch3_buflen + sizeof(MPID_nem_dcfa_pkt_prefix_t); + *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_ACCUMULATE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_ACCUMULATE); return mpi_errno; fn_fail: goto fn_exit; @@ -1498,16 +1498,16 @@ int MPID_nem_dcfa_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* packet handler for wrapper packet of MPIDI_CH3_PKT_GET */ /* see MPIDI_CH3_PktHandler_Get (in src/mpid/ch3/src/ch3u_rma_sync.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_Get +#define FUNCNAME MPID_nem_ib_PktHandler_Get #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */) { - MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; + MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_get_t *ch3_pkt = - (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1516,46 +1516,46 @@ int MPID_nem_dcfa_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_GET); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_GET); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GET); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GET); /* Update occupation status of local SR (send request) queue */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - dprintf("MPID_nem_dcfa_Pkthandler_Get,lsr_seq_num_tail=%d,get_pkt->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - dprintf("MPID_nem_dcfa_Pkthandler_Get,lsr_seq_num_tail updated to %d\n", - vc_dcfa->ibcom->lsr_seq_num_tail); - -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + dprintf("MPID_nem_ib_Pkthandler_Get,lsr_seq_num_tail=%d,get_pkt->seq_num_tail=%d\n", + vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + dprintf("MPID_nem_ib_Pkthandler_Get,lsr_seq_num_tail updated to %d\n", + vc_ib->ibcom->lsr_seq_num_tail); + +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,put,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, - &vc_dcfa->ibcom->lsr_seq_num_tail); - dprintf("pkthandler,put,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, + &vc_ib->ibcom->lsr_seq_num_tail); + dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ dprintf("pkthandler,get,send_progress\n"); fflush(stdout); - MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET + MPID_NEM_IB_SEND_PROGRESS_POLLINGSET /* fall back to the original handler */ /* we don't need to worry about the difference caused by embedding seq_num * because size of MPI-header of MPIDI_CH3_PKT_PUT equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_dcfa_iSendContig + * see MPID_nem_ib_iSendContig */ - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_dcfa_pkt_prefix_t); + MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_Get(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - *buflen = ch3_buflen + sizeof(MPID_nem_dcfa_pkt_prefix_t); + *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_GET); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GET); return mpi_errno; fn_fail: goto fn_exit; @@ -1564,16 +1564,16 @@ int MPID_nem_dcfa_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* packet handler for wrapper packet of MPIDI_CH3_PKT_GET_RESP */ /* see MPIDI_CH3_PktHandler_GetResp (in src/mpid/ch3/src/ch3u_rma_sync.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_GetResp +#define FUNCNAME MPID_nem_ib_PktHandler_GetResp #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen /* out */ , MPID_Request ** rreqp /* out */) { - MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; + MPID_nem_ib_pkt_prefix_t *netmod_pkt = (MPID_nem_ib_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_get_t *ch3_pkt = - (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_ib_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1582,46 +1582,46 @@ int MPID_nem_dcfa_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_GETRESP); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_GETRESP); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GETRESP); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GETRESP); /* Update occupation status of local SR (send request) queue */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - dprintf("MPID_nem_dcfa_Pkthandler_GetResp,lsr_seq_num_tail=%d,get_pkt->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); - dprintf("MPID_nem_dcfa_Pkthandler_GetResp,lsr_seq_num_tail updated to %d\n", - vc_dcfa->ibcom->lsr_seq_num_tail); - -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + dprintf("MPID_nem_ib_Pkthandler_GetResp,lsr_seq_num_tail=%d,get_pkt->seq_num_tail=%d\n", + vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, netmod_pkt->seq_num_tail); + dprintf("MPID_nem_ib_Pkthandler_GetResp,lsr_seq_num_tail updated to %d\n", + vc_ib->ibcom->lsr_seq_num_tail); + +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - dprintf("pkthandler,put,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, - &vc_dcfa->ibcom->lsr_seq_num_tail); - dprintf("pkthandler,put,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + dprintf("pkthandler,put,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, + &vc_ib->ibcom->lsr_seq_num_tail); + dprintf("pkthandler,put,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif dprintf("pkthandler,put,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ dprintf("pkthandler,get,send_progress\n"); fflush(stdout); - MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET + MPID_NEM_IB_SEND_PROGRESS_POLLINGSET /* fall back to the original handler */ /* we don't need to worry about the difference caused by embedding seq_num * because size of MPI-header of MPIDI_CH3_PKT_PUT equals to sizeof(MPIDI_CH3_Pkt_t) - * see MPID_nem_dcfa_iSendContig + * see MPID_nem_ib_iSendContig */ - MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_dcfa_pkt_prefix_t); + MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_GetResp(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - *buflen = ch3_buflen + sizeof(MPID_nem_dcfa_pkt_prefix_t); + *buflen = ch3_buflen + sizeof(MPID_nem_ib_pkt_prefix_t); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_GETRESP); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_GETRESP); return mpi_errno; fn_fail: goto fn_exit; @@ -1629,19 +1629,19 @@ int MPID_nem_dcfa_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* MPI_Isend set req-type to MPIDI_REQUEST_TYPE_RECV */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_pkt_GET_DONE_handler +#define FUNCNAME MPID_nem_ib_pkt_GET_DONE_handler #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_pkt_lmt_get_done_t *const done_pkt = (MPID_nem_dcfa_pkt_lmt_get_done_t *) pkt; + MPID_nem_ib_pkt_lmt_get_done_t *const done_pkt = (MPID_nem_ib_pkt_lmt_get_done_t *) pkt; MPID_Request *req; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKT_GET_DONE_HANDLER); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKT_GET_DONE_HANDLER); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKT_GET_DONE_HANDLER); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKT_GET_DONE_HANDLER); dprintf("get_done_handler,enter\n"); @@ -1652,7 +1652,7 @@ int MPID_nem_dcfa_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, switch (MPIDI_Request_get_type(req)) { /* MPIDI_Request_set_type is not performed when - * MPID_Isend --> FDU_or_AEP --> recv_posted --> dcfa_poll --> PUTCTS packet-handler */ + * MPID_Isend --> FDU_or_AEP --> recv_posted --> ib_poll --> PUTCTS packet-handler */ case MPIDI_REQUEST_TYPE_RECV: MPIU_ERR_INTERNALANDJUMP(mpi_errno, "unexpected request type"); break; @@ -1662,32 +1662,32 @@ int MPID_nem_dcfa_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, case MPIDI_REQUEST_TYPE_BSEND: /* extract embeded RDMA-write-to buffer occupancy information */ dprintf("get_done_handler,old lsr_seq_num_tail=%d,done_pkt->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, done_pkt->seq_num_tail); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, done_pkt->seq_num_tail); - //dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_dcfa->ibcom->lsr_seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail, done_pkt->seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, done_pkt->seq_num_tail); + //dprintf("lmt_start_recv,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail); -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, - &vc_dcfa->ibcom->lsr_seq_num_tail); - //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + //dprintf("lmt_start_recv,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, + &vc_ib->ibcom->lsr_seq_num_tail); + //dprintf("lmt_start_recv,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif - //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + //dprintf("lmt_start_recv,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ //dprintf("lmt_start_recv,reply_seq_num,send_progress\n"); - if (!MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)) { + if (!MPID_nem_ib_sendq_empty(vc_ib->sendq)) { dprintf("get_done_handler,ncom=%d,ncqe=%d,diff=%d(%d-%d)\n", - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG, - vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail); + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG, + vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail); } dprintf("get_done_handler,send_progress\n"); fflush(stdout); - MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS mpi_errno = vc->ch.lmt_done_send(vc, req); + MPID_NEM_IB_CHECK_AND_SEND_PROGRESS mpi_errno = vc->ch.lmt_done_send(vc, req); if (mpi_errno) MPIU_ERR_POP(mpi_errno); break; @@ -1700,25 +1700,25 @@ int MPID_nem_dcfa_pkt_GET_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, fn_exit: MPIU_THREAD_CS_EXIT(LMT,); - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKT_GET_DONE_HANDLER); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKT_GET_DONE_HANDLER); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_req_seq_num +#define FUNCNAME MPID_nem_ib_PktHandler_req_seq_num #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_pkt_req_seq_num_t *const req_pkt = (MPID_nem_dcfa_pkt_req_seq_num_t *) pkt; + MPID_nem_ib_pkt_req_seq_num_t *const req_pkt = (MPID_nem_ib_pkt_req_seq_num_t *) pkt; MPID_Request *req; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_REQ_SEQ_NUM); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_REQ_SEQ_NUM); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REQ_SEQ_NUM); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REQ_SEQ_NUM); /* mark as all of the message is read */ *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -1728,42 +1728,42 @@ int MPID_nem_dcfa_PktHandler_req_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, /* update occupancy info of SR */ /* request piggy-backs seq_num although it's requesting responder's seq_num */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - vc_dcfa->ibcom->lsr_seq_num_tail = - DCFA_MAX(vc_dcfa->ibcom->lsr_seq_num_tail, req_pkt->seq_num_tail); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + vc_ib->ibcom->lsr_seq_num_tail = + MPID_NEM_IB_MAX(vc_ib->ibcom->lsr_seq_num_tail, req_pkt->seq_num_tail); dprintf("PktHandler_req_seq_num,sendq=%d,ncom=%d,ncqe=%d,diff=%d(%d-%d)\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG, - vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail); + MPID_nem_ib_sendq_empty(vc_ib->sendq), + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG, + vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail); /* send reply */ - dprintf("PktHandler_req_seq_num,sending reply_seq_num,id=%d\n", vc_dcfa->ibcom->sseq_num); - MPID_nem_dcfa_send_reply_seq_num(vc); + dprintf("PktHandler_req_seq_num,sending reply_seq_num,id=%d\n", vc_ib->ibcom->sseq_num); + MPID_nem_ib_send_reply_seq_num(vc); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_REQ_SEQ_NUM); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REQ_SEQ_NUM); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_reply_seq_num +#define FUNCNAME MPID_nem_ib_PktHandler_reply_seq_num #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, +int MPID_nem_ib_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_pkt_reply_seq_num_t *const reply_pkt = (MPID_nem_dcfa_pkt_reply_seq_num_t *) pkt; + MPID_nem_ib_pkt_reply_seq_num_t *const reply_pkt = (MPID_nem_ib_pkt_reply_seq_num_t *) pkt; MPID_Request *req; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_REPLY_SEQ_NUM); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_REPLY_SEQ_NUM); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM); /* mark as all of the message is consumed */ @@ -1774,48 +1774,48 @@ int MPID_nem_dcfa_PktHandler_reply_seq_num(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pk /* update occupancy info of RDMA-write-buf */ int *lsr_seq_num_tail; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); dprintf("pkthandler,reply_seq_num,old lsr_seq_num=%d,reply_pkt->seq_num_tail=%d\n", - vc_dcfa->ibcom->lsr_seq_num_tail, reply_pkt->seq_num_tail); - ibcom_errno = ibcom_lsr_seq_num_tail_get(vc_dcfa->sc->fd, &lsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_lsr_seq_num_tail_get"); - *lsr_seq_num_tail = DCFA_MAX(*lsr_seq_num_tail, reply_pkt->seq_num_tail); - //dprintf("pkthandler,reply_seq_num,new lsr_seq_num=%d\n", vc_dcfa->ibcom->lsr_seq_num_tail); + vc_ib->ibcom->lsr_seq_num_tail, reply_pkt->seq_num_tail); + ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get"); + *lsr_seq_num_tail = MPID_NEM_IB_MAX(*lsr_seq_num_tail, reply_pkt->seq_num_tail); + //dprintf("pkthandler,reply_seq_num,new lsr_seq_num=%d\n", vc_ib->ibcom->lsr_seq_num_tail); -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE /* change remote notification policy of RDMA-write-to buf */ - //dprintf("pkthandler,reply_seq_num,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); - MPID_nem_dcfa_change_rdmabuf_occupancy_notify_policy_lw(vc_dcfa, lsr_seq_num_tail); - //dprintf("pkthandler,reply_seq_num,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + //dprintf("pkthandler,reply_seq_num,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); + MPID_nem_ib_change_rdmabuf_occupancy_notify_policy_lw(vc_ib, lsr_seq_num_tail); + //dprintf("pkthandler,reply_seq_num,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif - //dprintf("pkthandler,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + //dprintf("pkthandler,reply_seq_num,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* try to send from sendq because at least one RDMA-write-to buffer has been released */ //dprintf("pkthandler,reply_seq_num,send_progress\n"); dprintf("pkthandler,reply_seq_num,send_progress\n"); - MPID_NEM_DCFA_CHECK_AND_SEND_PROGRESS fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_REPLY_SEQ_NUM); + MPID_NEM_IB_CHECK_AND_SEND_PROGRESS fn_exit: + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_REPLY_SEQ_NUM); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_PktHandler_change_rdmabuf_occupancy_notify_state +#define FUNCNAME MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc, +int MPID_nem_ib_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * buflen, MPID_Request ** rreqp) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_pkt_change_rdmabuf_occupancy_notify_state_t *const reply_pkt = - (MPID_nem_dcfa_pkt_change_rdmabuf_occupancy_notify_state_t *) pkt; + MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t *const reply_pkt = + (MPID_nem_ib_pkt_change_rdmabuf_occupancy_notify_state_t *) pkt; MPID_Request *req; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_CHANGE_RDMABUF_OCCUPANCY_NOTIFY_STATE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_CHANGE_RDMABUF_OCCUPANCY_NOTIFY_STATE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_PKTHANDLER_CHANGE_RDMABUF_OCCUPANCY_NOTIFY_STATE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_PKTHANDLER_CHANGE_RDMABUF_OCCUPANCY_NOTIFY_STATE); /* mark as all of the message is read */ *buflen = sizeof(MPIDI_CH3_Pkt_t); @@ -1825,44 +1825,44 @@ int MPID_nem_dcfa_PktHandler_change_rdmabuf_occupancy_notify_state(MPIDI_VC_t * /* update occupancy info of SR */ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); dprintf("pkthandler,change notify state,old lstate=%d,pkt->state=%d\n", - vc_dcfa->ibcom->rdmabuf_occupancy_notify_lstate, reply_pkt->state); + vc_ib->ibcom->rdmabuf_occupancy_notify_lstate, reply_pkt->state); int *rdmabuf_occupancy_notify_lstate; ibcom_errno = - ibcom_rdmabuf_occupancy_notify_lstate_get(vc_dcfa->sc->fd, + MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get(vc_ib->sc->fd, &rdmabuf_occupancy_notify_lstate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**ibcom_rdmabuf_occupancy_notify_lstate_get"); + "**MPID_nem_ib_com_rdmabuf_occupancy_notify_lstate_get"); *rdmabuf_occupancy_notify_lstate = reply_pkt->state; dprintf("pkthandler,change notify state,new lstate=%d\n", - vc_dcfa->ibcom->rdmabuf_occupancy_notify_lstate); + vc_ib->ibcom->rdmabuf_occupancy_notify_lstate); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_CHANGE_RDMABUF_OCCUPANCY_NOTIFY_STATE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_PKTHANDLER_CHANGE_RDMABUF_OCCUPANCY_NOTIFY_STATE); return mpi_errno; fn_fail: goto fn_exit; } -#ifdef DCFA_ONDEMAND +#ifdef MPID_NEM_IB_ONDEMAND #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_cm_drain_scq +#define FUNCNAME MPID_nem_ib_cm_drain_scq #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_cm_drain_scq() +int MPID_nem_ib_cm_drain_scq() { int mpi_errno = MPI_SUCCESS; int result; int i; - struct ibv_wc cqe[IBCOM_MAX_CQ_HEIGHT_DRAIN]; + struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_CM_DRAIN_SCQ); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_CM_DRAIN_SCQ); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ); - result = ibv_poll_cq(rc_shared_scq_lmt_put, IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); - MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,dcfa,ibv_poll_cq"); + result = ibv_poll_cq(MPID_nem_ib_rc_shared_scq_lmt_put, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe[0]); + MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { dprintf("cm_drain_scq,found,result=%d\n", result); @@ -1880,140 +1880,140 @@ int MPID_nem_dcfa_cm_drain_scq() } #endif MPIU_ERR_CHKANDJUMP(cqe[i].status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_cm_drain_scq"); + "**MPID_nem_ib_cm_drain_scq"); /* TODO retry a connection request when it timed out */ - if (cqe.wr_id == MPID_NEM_DCFA_SYN || cqe.wr_id == MPID_NEM_DCFA_SYNACK) { + if (cqe.wr_id == MPID_NEM_IB_SYN || cqe.wr_id == MPID_NEM_IB_SYNACK) { - MPID_nem_dcfa_conn_ud_ibcom->ncom_lmt_put -= 1; - MPID_nem_dcfa_ncqe_connect -= 1; + MPID_nem_ib_conn_ud_ibcom->ncom_lmt_put -= 1; + MPID_nem_ib_ncqe_connect -= 1; /* Try to send from sendq_connect */ - if (!MPID_nem_dcfa_sendq_empty(sendq_connect) && - MPID_nem_dcfa_ncom_lmt_put < IBCOM_MAX_SQ_CAPACITY && - MPID_nem_dcfa_ncqe_lmt_put < IBCOM_MAX_CQ_CAPACITY) { - MPID_nem_dcfa_send_progress_connect(); + if (!MPID_nem_ib_sendq_empty(sendq_connect) && + MPID_nem_ib_ncom_lmt_put < MPID_NEM_IB_COM_MAX_SQ_CAPACITY && + MPID_nem_ib_ncqe_lmt_put < MPID_NEM_IB_COM_MAX_CQ_CAPACITY) { + MPID_nem_ib_send_progress_connect(); } } else { printf("unknown command=%d\n", cqe.wr_id); - MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_cm_drain_scq"); + MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_drain_scq"); } } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_CM_DRAIN_SCQ); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_CM_DRAIN_SCQ); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_cm_poll +#define FUNCNAME MPID_nem_ib_cm_poll #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -int MPID_nem_dcfa_cm_poll() +int MPID_nem_ib_cm_poll() { int mpi_errno = MPI_SUCCESS; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_CM_POLL); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_CM_POLL); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CM_POLL); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CM_POLL); dprintf("cm_poll,enter\n"); - volatile uint32_t *owner = (uint32_t *) (ibcom_scratch_pad->icom_mem[IBCOM_SCRATCH_PAD_TO]); + volatile uint32_t *owner = (uint32_t *) (MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO]); if (*owner == (uint32_t) - 1) { goto fn_exit; } /* not acquired */ - IbCom *ibcom_scratch_pad; - ibcom_errno = ibcom_obtain_pointer(MPID_nem_dcfa_scratch_pad_fds[*owner], &ibcom_scratch_pad); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); + MPID_nem_ib_com *ibcom_scratch_pad; + ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[*owner], &ibcom_scratch_pad); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); - MPID_nem_dcfa_cm_cmd_t *received = - (MPID_nem_dcfa_cm_cmd_t *) (ibcom_scratch_pad->icom_mem[IBCOM_SCRATCH_PAD_TO] + + MPID_nem_ib_cm_cmd_t *received = + (MPID_nem_ib_cm_cmd_t *) (ibcom_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + sizeof(uint32_t)); - MPID_nem_dcfa_cm_cmd_t cmd; - MPID_nem_dcfa_vc_area *vc_dcfa; + MPID_nem_ib_cm_cmd_t cmd; + MPID_nem_ib_vc_area *vc_ib; switch (received->type) { - case MPID_NEM_DCFA_CM_SYN: - ibcom_errno = ibcomOpen(ib_port, IBCOM_OPEN_RC, &MPID_nem_dcfa_conns[*owner].fd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcomOpen"); - cmd.type = MPID_NEM_DCFA_CM_SYNACK; + case MPID_NEM_IB_CM_SYN: + ibcom_errno = MPID_nem_ib_com_open(ib_port, MPID_NEM_IB_COM_OPEN_RC, &MPID_nem_ib_conns[*owner].fd); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_open"); + cmd.type = MPID_NEM_IB_CM_SYNACK; goto common_tail; break; - case MPID_NEM_DCFA_CM_BUSINESSCARD: + case MPID_NEM_IB_CM_BUSINESSCARD: ibcom_errno = - ibcom_rts(MPID_nem_dcfa_conns[*owner].fd, received->qpnum, received->lid, + MPID_nem_ib_com_rts(MPID_nem_ib_conns[*owner].fd, received->qpnum, received->lid, &(received->gid)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rts"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rts"); ibcom_errno = - ibcom_reg_mr_connect(MPID_nem_dcfa_conns[*owner].fd, received->rmem, received->rkey); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_reg_mr_connect"); - VC_FIELD(MPID_nem_dcfa_conns[owner].vc, is_connected) = 1; + MPID_nem_ib_com_reg_mr_connect(MPID_nem_ib_conns[*owner].fd, received->rmem, received->rkey); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_connect"); + VC_FIELD(MPID_nem_ib_conns[owner].vc, is_connected) = 1; - cmd.type = MPID_NEM_DCFA_CM_ACK; + cmd.type = MPID_NEM_IB_CM_ACK; common_tail: ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[*owner].fd, IBCOM_INFOKEY_PORT_LID, &(cmd.lid), + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_PORT_LID, &(cmd.lid), sizeof(uint16_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[*owner].fd, IBCOM_INFOKEY_PORT_GID, &(cmd.gid), + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_PORT_GID, &(cmd.gid), sizeof(union ibv_gid)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[*owner].fd, IBCOM_INFOKEY_QP_QPN, &(cmd.qpnum), + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &(cmd.qpnum), sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_conns[*owner].fd, IBCOM_SCRATCH_PAD_TO, - IBCOM_INFOKEY_MR_ADDR, &(cmd.rmem), sizeof(void *)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &(cmd.rmem), sizeof(void *)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_conns[*owner].fd, IBCOM_SCRATCH_PAD_TO, - IBCOM_INFOKEY_MR_RKEY, &(cmd.rkey), sizeof(int)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[*owner].fd, MPID_NEM_IB_COM_SCRATCH_PAD_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &(cmd.rkey), sizeof(int)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); *owner = (uint32_t) - 1; /* release */ - mpi_errno = MPID_nem_dcfa_cm_send_core(rank, &cmd); - MPIU_ERR_CHKANDJUMP(mp_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_cm_send_core"); + mpi_errno = MPID_nem_ib_cm_send_core(rank, &cmd); + MPIU_ERR_CHKANDJUMP(mp_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_send_core"); break; default: printf("unknown connection command\n"); - MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_cm_poll"); + MPIU_ERR_CHKANDJUMP(1, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_poll"); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_CM_POLL); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_CM_POLL); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_cm_accept +#define FUNCNAME MPID_nem_ib_cm_accept #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_cm_accept() +int MPID_nem_ib_cm_accept() { int mpi_errno = MPI_SUCCESS; int ibcom_errno; int result; int i; - struct ibv_wc cqe[IBCOM_MAX_CQ_HEIGHT_DRAIN]; + struct ibv_wc cqe[MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN]; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_CM_ACCEPT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_CM_ACCEPT); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CM_ACCEPT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CM_ACCEPT); - result = ibv_poll_cq(ud_shared_rcq, IBCOM_MAX_CQ_HEIGHT_DRAIN, &cqe); - MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,dcfa,ibv_poll_cq"); + result = ibv_poll_cq(MPID_nem_ib_ud_shared_rcq, MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN, &cqe); + MPIU_ERR_CHKANDJUMP(result < 0, mpi_errno, MPI_ERR_OTHER, "**netmod,ib,ibv_poll_cq"); if (result > 0) { dprintf("accept,result=%d\n", result); @@ -2022,49 +2022,49 @@ int MPID_nem_dcfa_cm_accept() dprintf("accept,i=%d\n", i); MPIU_ERR_CHKANDJUMP(cqe.status != IBV_WC_SUCCESS, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_cm_accept"); + "**MPID_nem_ib_cm_accept"); void *rbuf; - ibcom_errno = ibcom_mem_udwr_to(MPID_nem_dcfa_conn_ud_fd, &rbuf); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_mem_udwr_to"); - MPID_nem_dcfa_conn_pkt_t *rpkt = (MPID_nem_dcfa_conn_pkt_t *) (rbuf + 40); - if (rpkt->type == MPID_NEM_DCFA_SYN) { + ibcom_errno = MPID_nem_ib_com_mem_udwr_to(MPID_nem_ib_conn_ud_fd, &rbuf); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_mem_udwr_to"); + MPID_nem_ib_conn_pkt_t *rpkt = (MPID_nem_ib_conn_pkt_t *) (rbuf + 40); + if (rpkt->type == MPID_NEM_IB_SYN) { - dprintf("accept,%d<-%d,type=%08x\n", MPID_nem_dcfa_myrank, rpkt->remote_rank, + dprintf("accept,%d<-%d,type=%08x\n", MPID_nem_ib_myrank, rpkt->remote_rank, rpkt->type); void *sbuf; - ibcom_errno = ibcom_mem_udwr_from(MPID_nem_dcfa_conn_ud_fd, &sbuf); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_mem_udwr_from"); - MPID_nem_dcfa_conn_pkt_t *spkt = (MPID_nem_dcfa_conn_pkt_t *) (sbuf + 40); - spkt->remote_rank = MPID_nem_dcfa_myrank; - spkt->type = MPID_NEM_DCFA_SYNACK; + ibcom_errno = MPID_nem_ib_com_mem_udwr_from(MPID_nem_ib_conn_ud_fd, &sbuf); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_mem_udwr_from"); + MPID_nem_ib_conn_pkt_t *spkt = (MPID_nem_ib_conn_pkt_t *) (sbuf + 40); + spkt->remote_rank = MPID_nem_ib_myrank; + spkt->type = MPID_NEM_IB_SYNACK; ibcom_errno = - ibcom_get_info_conn(MPID_nem_dcfa_conns[rpkt->remote_rank].fd, IBCOM_INFOKEY_QP_QPN, + MPID_nem_ib_com_get_info_conn(MPID_nem_ib_conns[rpkt->remote_rank].fd, MPID_NEM_IB_COM_INFOKEY_QP_QPN, &spkt->qpn, sizeof(uint32_t)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_conn"); ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_conns[remote_rank].fd, IBCOM_RDMAWR_TO, - IBCOM_INFOKEY_MR_ADDR, &spkt->rmem, sizeof(void *)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, MPID_NEM_IB_COM_RDMAWR_TO, + MPID_NEM_IB_COM_INFOKEY_MR_ADDR, &spkt->rmem, sizeof(void *)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); ibcom_errno = - ibcom_get_info_mr(MPID_nem_dcfa_conns[remote_rank].fd, IBCOM_RDMAWR_TO, - IBCOM_INFOKEY_MR_RKEY, &spkt->rkey, sizeof(int)); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_mr"); + MPID_nem_ib_com_get_info_mr(MPID_nem_ib_conns[remote_rank].fd, MPID_NEM_IB_COM_RDMAWR_TO, + MPID_NEM_IB_COM_INFOKEY_MR_RKEY, &spkt->rkey, sizeof(int)); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_get_info_mr"); /* kokomade. add udsend_core(synack) */ - if (MPID_nem_dcfa_conn_ibcom->ncom < &&MPID_nem_dcfa_ncqe_connect <) { - MPID_nem_dcfa_conn_send_core(rpkt->remote_rank); + if (MPID_nem_ib_conn_ibcom->ncom < &&MPID_nem_ib_ncqe_connect <) { + MPID_nem_ib_conn_send_core(rpkt->remote_rank); } else { - MPID_nem_dcfa_sendq_conn_entry_t *entry = - MPIU_Malloc(sizeof(MPID_nem_dcfa_sendq_conn_entry_t)); + MPID_nem_ib_sendq_conn_entry_t *entry = + MPIU_Malloc(sizeof(MPID_nem_ib_sendq_conn_entry_t)); MPIU_ERR_CHKANDJUMP(!entry, mpi_errno, MPI_ERR_OTHER, "**outofmemory"); entry->pending_pkt = *spkt; - MPID_nem_dcfa_conn_sendq_enqueue(MPID_nem_dcfa_conn_sendq, entry); + MPID_nem_ib_conn_sendq_enqueue(MPID_nem_ib_conn_sendq, entry); } @@ -2074,7 +2074,7 @@ int MPID_nem_dcfa_cm_accept() } } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_CM_ACCEPT); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_CM_ACCEPT); return mpi_errno; fn_fail: goto fn_exit; diff --git a/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c new file mode 100644 index 0000000..145ae53 --- /dev/null +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c @@ -0,0 +1,321 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2012 NEC Corporation + * Author: Masamichi Takagi + * (C) 2012 Oct 10 Min Si + * (C) 2001-2009 Yutaka Ishikawa + * See COPYRIGHT in top-level directory. + */ + +#include <unistd.h> +#include <stdlib.h> +#include "ib_ibcom.h" + +//#define MPID_NEM_IB_DEBUG_REG_MR +#ifdef MPID_NEM_IB_DEBUG_REG_MR +#define dprintf printf +#else +#define dprintf(...) +#endif + +/* cache size of ibv_reg_mr */ +#define MPID_NEM_IB_COM_REG_MR_NLINE 4096 +#define MPID_NEM_IB_COM_REG_MR_NWAY 1024 + +#define MPID_NEM_IB_COM_REG_MR_SZPAGE 4096 +#define MPID_NEM_IB_COM_REG_MR_LOGSZPAGE 12 + +/* arena allocator */ + +#define MPID_NEM_IB_NIALLOCID 32 +typedef struct { + char *next; +} free_list_t; +static char *free_list_front[MPID_NEM_IB_NIALLOCID] = { 0 }; +static char *arena_flist[MPID_NEM_IB_NIALLOCID] = { 0 }; + +#define MPID_NEM_IB_SZARENA 4096 +#define MPID_NEM_IB_CLUSTER_SIZE (MPID_NEM_IB_SZARENA/sz) +#define MPID_NEM_IB_ROUNDUP64(addr, align) ((addr + align - 1) & ~((unsigned long)align - 1)) +#define MPID_NEM_IB_NCLUST_SLAB 1 +#define MPID_NEM_IB_COM_AALLOC_ID_MRCACHE 0 + +static inline void *aalloc(size_t sz, int id) +{ +#if 1 /* debug */ + return malloc(sz); +#else + char *p = free_list_front[id]; + if ((unsigned long) p & (MPID_NEM_IB_SZARENA - 1)) { + free_list_front[id] += sz; + return p; + } + else { + char *q, r; + if (arena_flist[id]) { + q = arena_flist[id]; + arena_flist[id] = ((free_list_t *) arena_flist[id])->next; + } + else { + q = mmap(NULL, MPID_NEM_IB_ROUNDUP64(MPID_NEM_IB_SZARENA * MPID_NEM_IB_NCLUST_SLAB, 4096), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); +#if MPID_NEM_IB_NCLUST_SLAB > 1 + arena_flist[id] = q + MPID_NEM_IB_SZARENA; + for (p = arena_flist[id]; p < q + (MPID_NEM_IB_NCLUST_SLAB - 1) * MPID_NEM_IB_SZARENA; p += MPID_NEM_IB_SZARENA) { + ((free_list_t *) p)->next = p + MPID_NEM_IB_SZARENA; + } + ((free_list_t *) p)->next = 0; +#endif + } + *((int *) q) = MPID_NEM_IB_CLUSTER_SIZE - 1; + // dprintf("q=%llx\n", q); + q += sz + (MPID_NEM_IB_SZARENA % sz); + free_list_front[id] = q + sz; + return q; + } +#endif +} + +static inline void afree(const void *p, int id) +{ +#if 1 /* debug */ + return free((void *) p); +#else + p = (void *) ((unsigned long) p & ~(MPID_NEM_IB_SZARENA - 1)); + if (!(--(*((int *) p)))) { + ((free_list_t *) p)->next = arena_flist[id]; + arena_flist[id] = (char *) p; + } +#endif +} + +struct MPID_nem_ib_com_reg_mr_listnode_t { + struct MPID_nem_ib_com_reg_mr_listnode_t *lru_next; + struct MPID_nem_ib_com_reg_mr_listnode_t *lru_prev; +}; + +struct MPID_nem_ib_com_reg_mr_cache_entry_t { + /* : public MPID_nem_ib_com_reg_mr_listnode_t */ + struct MPID_nem_ib_com_reg_mr_listnode_t *lru_next; + struct MPID_nem_ib_com_reg_mr_listnode_t *lru_prev; + + struct ibv_mr *mr; + void *addr; + int len; + int refc; +}; + +static struct MPID_nem_ib_com_reg_mr_listnode_t MPID_nem_ib_com_reg_mr_cache[MPID_NEM_IB_COM_REG_MR_NLINE]; + +__inline__ int MPID_nem_ib_com_hash_func(char *addr) +{ + unsigned int v = (unsigned int) (unsigned long) addr; + //v = v >> MPID_NEM_IB_COM_REG_MR_LOGSZPAGE; /* assume it is page aligned */ + v = v & (MPID_NEM_IB_COM_REG_MR_NLINE - 1); + return (int) v; +} + +static void MPID_nem_ib_com_reg_mr_insert(struct MPID_nem_ib_com_reg_mr_listnode_t *c, struct MPID_nem_ib_com_reg_mr_listnode_t *e) +{ + struct MPID_nem_ib_com_reg_mr_listnode_t *next; + struct MPID_nem_ib_com_reg_mr_listnode_t *prev; + prev = c; + next = prev->lru_next; + e->lru_next = next; + e->lru_prev = prev; + next->lru_prev = e; + prev->lru_next = e; +} + +static void MPID_nem_ib_com_reg_mr_unlink(struct MPID_nem_ib_com_reg_mr_listnode_t *e) +{ + struct MPID_nem_ib_com_reg_mr_listnode_t *next, *prev; + next = e->lru_next; + prev = e->lru_prev; + next->lru_prev = prev; + prev->lru_next = next; +} + +static inline void __lru_queue_display() +{ + struct MPID_nem_ib_com_reg_mr_cache_entry_t *p; + int i = 0; + for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { + dprintf("---- hash %d\n", i); + for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next; + p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i]; + p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) { + if (p && p->addr) { + dprintf("-------- p=%p,addr=%p,len=%d,refc=%d,lru_next=%p\n", p, p->addr, p->len, + p->refc, p->lru_next); + } + else { + dprintf("-------- p=%p,lru_next=%p\n", p, p->lru_next); + } + } + } +} + +struct ibv_mr *MPID_nem_ib_com_reg_mr_fetch(void *addr, int len) +{ +#if 0 /* debug */ + struct ibv_mr *mr; + int ibcom_errno = MPID_nem_ib_com_reg_mr(addr, len, &mr); + printf("mrcache,MPID_nem_ib_com_reg_mr,error,addr=%p,len=%d,lkey=%08x,rkey=%08x\n", addr, len, mr->lkey, + mr->rkey); + if (ibcom_errno != 0) { + goto fn_fail; + } + fn_exit: + return mr; + fn_fail: + goto fn_exit; +#else + int ibcom_errno; + int key; + struct MPID_nem_ib_com_reg_mr_cache_entry_t *e; + +#if 1 /*def HAVE_LIBDCFA */ + /* we can't change addr because ibv_post_send assumes mr->host_addr (output of this function) + * must have an exact mirror value of addr (input of this function) */ + void *addr_aligned = addr; + int len_aligned = len; +#else + void *addr_aligned = (void *) ((unsigned long) addr & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); + int len_aligned = + ((((unsigned long) addr + len) - (unsigned long) addr_aligned + MPID_NEM_IB_COM_REG_MR_SZPAGE - + 1) & ~(MPID_NEM_IB_COM_REG_MR_SZPAGE - 1)); +#endif + key = MPID_nem_ib_com_hash_func(addr); + + dprintf("[MrCache] addr=%p, len=%d\n", addr, len); + dprintf("[MrCache] aligned addr=%p, len=%d\n", addr_aligned, len_aligned); + + //__lru_queue_display(); + int way = 0; + for (e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next; + e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[key]; + e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) e->lru_next, way++) { + //dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next); + + if (e->addr <= addr_aligned && (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) { + //dprintf + //("MPID_nem_ib_com_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n", + //e->addr, e->len, e->mr->addr, e->mr->length, addr, len); + goto hit; + } + } + + // miss + + // evict an entry and de-register its MR when the cache-set is full + if (way > MPID_NEM_IB_COM_REG_MR_NWAY) { + struct MPID_nem_ib_com_reg_mr_cache_entry_t *victim = + (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) e->lru_prev; + MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *) victim); + + //dprintf("MPID_nem_ib_com_reg_mr,evict,entry addr=%p,len=%d,mr addr=%p,len=%ld\n", e->addr, e->len, + //e->mr->addr, e->mr->length); + ibcom_errno = MPID_nem_ib_com_dereg_mr(victim->mr); + if (ibcom_errno) { + printf("mrcache,MPID_nem_ib_com_dereg_mr\n"); + goto fn_fail; + } + afree(victim, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); + } + + e = aalloc(sizeof(struct MPID_nem_ib_com_reg_mr_cache_entry_t), MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); + /* reference counter is used when evicting entry */ + e->refc = 1; + + dprintf("MPID_nem_ib_com_reg_mr_fetch,miss,addr=%p,len=%d\n", addr_aligned, len_aligned); + /* register memory */ + ibcom_errno = MPID_nem_ib_com_reg_mr(addr_aligned, len_aligned, &e->mr); + if (ibcom_errno != 0) { + fprintf(stderr, "mrcache,MPID_nem_ib_com_reg_mr\n"); + goto fn_fail; + } + e->addr = addr_aligned; + e->len = len_aligned; + + //dprintf("MPID_nem_ib_com_reg_mr_fetch,fill,e=%p,key=%d,mr=%p,mr addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, + //key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); + + /* register to cache */ + MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); + + //__lru_queue_display(); + + goto fn_exit; + + hit: + + /* reference counter is used when evicting entry */ + e->refc++; +#if 0 /* disable for debug */ + /* move to head of the list */ + if (e != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[key].lru_next) { + MPID_nem_ib_com_reg_mr_unlink((struct MPID_nem_ib_com_reg_mr_listnode_t *) e); + MPID_nem_ib_com_reg_mr_insert(&MPID_nem_ib_com_reg_mr_cache[key], (struct MPID_nem_ib_com_reg_mr_listnode_t *) e); + } +#endif + //dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, + //key, e->mr, e->refc, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); + + //__lru_queue_display(); + + fn_exit: + return e->mr; + fn_fail: + goto fn_exit; +#endif +} + +static void MPID_nem_ib_com_reg_mr_dereg(struct ibv_mr *mr) +{ + + struct MPID_nem_ib_com_reg_mr_cache_entry_t *e; + struct MPID_nem_ib_com_reg_mr_cache_entry_t *zero = 0; + unsigned long offset = (unsigned long) zero->mr; + e = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) ((unsigned long) mr - offset); + e->refc--; + + //dprintf("MPID_nem_ib_com_reg_mr_dereg,entry=%p,mr=%p,addr=%p,refc=%d,offset=%lx\n", e, mr, e->mr->addr, + //e->refc, offset); +} + +void MPID_nem_ib_com_register_cache_init() +{ + int i; + + /* Using the address to the start node to express the end of the list + * instead of using NULL */ + for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { + MPID_nem_ib_com_reg_mr_cache[i].lru_next = (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i]; + MPID_nem_ib_com_reg_mr_cache[i].lru_prev = (struct MPID_nem_ib_com_reg_mr_listnode_t *) &MPID_nem_ib_com_reg_mr_cache[i]; + } + + dprintf("[MrCache] cache initializes %d entries\n", MPID_NEM_IB_COM_REG_MR_NLINE); +} + +void MPID_nem_ib_com_register_cache_destroy() +{ + struct MPID_nem_ib_com_reg_mr_cache_entry_t *p; + int i = 0, cnt = 0; + + for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) { + for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) MPID_nem_ib_com_reg_mr_cache[i].lru_next; + p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i]; + p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) { + if (p && p->addr > 0) { + MPID_nem_ib_com_dereg_mr(p->mr); + afree(p, MPID_NEM_IB_COM_AALLOC_ID_MRCACHE); + cnt++; + } + } + } + + //__lru_queue_display(); + + dprintf("[MrCache] cache destroyed %d entries\n", cnt); +} diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c similarity index 58% rename from src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c rename to src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c index dd6f120..e61c3cd 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c @@ -5,13 +5,13 @@ * See COPYRIGHT in top-level directory. */ -#include "dcfa_impl.h" +#include "ib_impl.h" -//#define DEBUG_DCFA_SEND +//#define MPID_NEM_IB_DEBUG_SEND #ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ #undef dprintf #endif -#ifdef DEBUG_DCFA_SEND +#ifdef MPID_NEM_IB_DEBUG_SEND #define dprintf printf #else #define dprintf(...) @@ -19,81 +19,81 @@ static int entered_send_progress = 0; -#ifdef DCFA_ONDEMAND +#ifdef MPID_NEM_IB_ONDEMAND // tree format is // one or more <left_pointer(int), right_pointer(int), value(int), length(int), string(char[])> -#define MPID_NEM_DCFA_LPTR(ptr) *(int*)((ptr) + sizeof(int)*0) -#define MPID_NEM_DCFA_RPTR(ptr) *(int*)((ptr) + sizeof(int)*1) -#define MPID_NEM_DCFA_VAL(ptr) *(int*)((ptr) + sizeof(int)*2) -#define MPID_NEM_DCFA_LEN(ptr) *(int*)((ptr) + sizeof(int)*3) -#define MPID_NEM_DCFA_PBODY(ptr) ((ptr) + sizeof(int)*4) +#define MPID_NEM_IB_MAP_LPTR(ptr) *(int*)((ptr) + sizeof(int)*0) +#define MPID_NEM_IB_MAP_RPTR(ptr) *(int*)((ptr) + sizeof(int)*1) +#define MPID_NEM_IB_MAP_VAL(ptr) *(int*)((ptr) + sizeof(int)*2) +#define MPID_NEM_IB_MAP_LEN(ptr) *(int*)((ptr) + sizeof(int)*3) +#define MPID_NEM_IB_MAP_PBODY(ptr) ((ptr) + sizeof(int)*4) -#define ALLOCATE(map, key, key_length, initial) { \ +#define MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, initial) { \ if (map->length + key_length + sizeof(int)*4 > map->max_length) { \ map->max_length = map->max_length ? map->max_length * 2 : 4096; \ map->data = realloc(map->data, map->max_length); \ } \ char* new_str = map->data + map->length; \ - MPID_NEM_DCFA_LPTR(new_str) = 0; \ - MPID_NEM_DCFA_RPTR(new_str) = 0; \ - MPID_NEM_DCFA_VAL(new_str) = initial; \ - MPID_NEM_DCFA_LEN(new_str) = key_length; \ - memcpy(MPID_NEM_DCFA_PBODY(new_str), key, key_length); \ + MPID_NEM_IB_MAP_LPTR(new_str) = 0; \ + MPID_NEM_IB_MAP_RPTR(new_str) = 0; \ + MPID_NEM_IB_MAP_VAL(new_str) = initial; \ + MPID_NEM_IB_MAP_LEN(new_str) = key_length; \ + memcpy(MPID_NEM_IB_MAP_PBODY(new_str), key, key_length); \ map->length += sizeof(int)*4 + key_length; \ } -void MPID_nem_dcfa_cm_map_set(MPID_nem_dcfa_cm_map_t * map, char *key, int key_length, int val) +void MPID_nem_ib_cm_map_set(MPID_nem_ib_cm_map_t * map, char *key, int key_length, int val) { char *pTree = map->data; - dprintf("MPID_nem_dcfa_cm_map_set,val=%d\n", val); + dprintf("MPID_nem_ib_cm_map_set,val=%d\n", val); if (!pTree) { - ALLOCATE(map, key, key_length, val); + MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); dprintf("pTree was empty\n"); return; } int s1_minus_s2; while (1) { - int lmin = key_length < MPID_NEM_DCFA_LEN(pTree) ? key_length : MPID_NEM_DCFA_LEN(pTree); - int residual = key_length - MPID_NEM_DCFA_LEN(pTree); - s1_minus_s2 = memcmp(key, MPID_NEM_DCFA_PBODY(pTree), lmin); + int lmin = key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); + int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree); + s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin); if (!s1_minus_s2 && !residual) { - MPID_NEM_DCFA_VAL(pTree) = val; + MPID_NEM_IB_MAP_VAL(pTree) = val; dprintf("found\n"); return; // same string, same length } else if (s1_minus_s2 < 0 || !s1_minus_s2 && residual < 0) { // psArg is "smaller" OR same substring, psArg is shorter - if (MPID_NEM_DCFA_LPTR(pTree) == 0) { - MPID_NEM_DCFA_LPTR(pTree) = map->length; // pointer write + if (MPID_NEM_IB_MAP_LPTR(pTree) == 0) { + MPID_NEM_IB_MAP_LPTR(pTree) = map->length; // pointer write /* left child */ - ALLOCATE(map, key, key_length, val); + MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); dprintf("stored as left child\n"); return; } - pTree = map->data + MPID_NEM_DCFA_LPTR(pTree); // go to left child + pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child } else { // psArg is "larger" OR same substring, psArg is longer - if (MPID_NEM_DCFA_RPTR(pTree) == 0) { - MPID_NEM_DCFA_RPTR(pTree) = map->length; // pointer write + if (MPID_NEM_IB_MAP_RPTR(pTree) == 0) { + MPID_NEM_IB_MAP_RPTR(pTree) = map->length; // pointer write /* right child */ - ALLOCATE(map, key, key_length, val); + MPID_NEM_IB_MAP_ALLOCATE(map, key, key_length, val); dprintf("stored as right child\n"); return; } - pTree = map->data + MPID_NEM_DCFA_RPTR(pTree); // go to right child + pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child } } } -int MPID_nem_dcfa_cm_map_get(MPID_nem_dcfa_cm_map_t * map, char *key, int key_length, int *val) +int MPID_nem_ib_cm_map_get(MPID_nem_ib_cm_map_t * map, char *key, int key_length, int *val) { int llc_errno = LLC_SUCCESS; char *pTree = map->data; - dprintf("MPID_nem_dcfa_cm_map_get,key=%s\n", key); + dprintf("MPID_nem_ib_cm_map_get,key=%s\n", key); if (!pTree) { llc_errno = -1; @@ -102,32 +102,32 @@ int MPID_nem_dcfa_cm_map_get(MPID_nem_dcfa_cm_map_t * map, char *key, int key_le } int s1_minus_s2; while (1) { - int lmin = key_length < MPID_NEM_DCFA_LEN(pTree) ? key_length : MPID_NEM_DCFA_LEN(pTree); - int residual = key_length - MPID_NEM_DCFA_LEN(pTree); - s1_minus_s2 = memcmp(key, MPID_NEM_DCFA_PBODY(pTree), lmin); + int lmin = key_length < MPID_NEM_IB_MAP_LEN(pTree) ? key_length : MPID_NEM_IB_MAP_LEN(pTree); + int residual = key_length - MPID_NEM_IB_MAP_LEN(pTree); + s1_minus_s2 = memcmp(key, MPID_NEM_IB_MAP_PBODY(pTree), lmin); if (!s1_minus_s2 && !residual) { - *val = MPID_NEM_DCFA_VAL(pTree); + *val = MPID_NEM_IB_MAP_VAL(pTree); dprintf("value found=%d\n", *val); goto fn_exit; // same string, same length } else if (s1_minus_s2 < 0 || !s1_minus_s2 && residual < 0) { // psArg is "smaller" OR same substring, psArg is shorter - if (MPID_NEM_DCFA_LPTR(pTree) == 0) { + if (MPID_NEM_IB_MAP_LPTR(pTree) == 0) { llc_errno = -1; dprintf("left is null\n"); goto fn_fail; } - pTree = map->data + MPID_NEM_DCFA_LPTR(pTree); // go to left child + pTree = map->data + MPID_NEM_IB_MAP_LPTR(pTree); // go to left child } else { // psArg is "larger" OR same substring, psArg is longer - if (MPID_NEM_DCFA_RPTR(pTree) == 0) { + if (MPID_NEM_IB_MAP_RPTR(pTree) == 0) { llc_errno = -1; dprintf("right is null\n"); goto fn_fail; } - pTree = map->data + MPID_NEM_DCFA_RPTR(pTree); // go to right child + pTree = map->data + MPID_NEM_IB_MAP_RPTR(pTree); // go to right child } } fn_exit: @@ -138,69 +138,69 @@ int MPID_nem_dcfa_cm_map_get(MPID_nem_dcfa_cm_map_t * map, char *key, int key_le #endif #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_iSendContig_core +#define FUNCNAME MPID_nem_ib_iSendContig_core #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPID_nem_dcfa_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, +static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); - MPID_nem_dcfa_pkt_prefix_t pkt_netmod; + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); + MPID_nem_ib_pkt_prefix_t pkt_netmod; void *netmod_hdr; int sz_netmod_hdr; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_ISENDCONTIG_CORE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_ISENDCONTIG_CORE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ISENDCONTIG_CORE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_ISENDCONTIG_CORE); /* piggy-back SR occupancy info might copy and modify given header */ /* remote SR sequence number which is last sent */ int *rsr_seq_num_tail; - ibcom_errno = ibcom_rsr_seq_num_tail_get(vc_dcfa->sc->fd, &rsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rsr_seq_num_tail_get"); + ibcom_errno = MPID_nem_ib_com_rsr_seq_num_tail_get(vc_ib->sc->fd, &rsr_seq_num_tail); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_rsr_seq_num_tail_get"); /* remote SR sequence number which is last sent */ int *rsr_seq_num_tail_last_sent; ibcom_errno = - ibcom_rsr_seq_num_tail_last_sent_get(vc_dcfa->sc->fd, &rsr_seq_num_tail_last_sent); + MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get(vc_ib->sc->fd, &rsr_seq_num_tail_last_sent); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**ibcom_rsr_seq_num_tail_last_sent_get"); + "**MPID_nem_ib_com_rsr_seq_num_tail_last_sent_get"); //dprintf("isendcontig,rsr_seq_num_tail=%d,rsr_seq_num_tail_last_sent=%d\n", *rsr_seq_num_tail, *rsr_seq_num_tail_last_sent); int notify_rate; - ibcom_errno = ibcom_rdmabuf_occupancy_notify_rate_get(vc_dcfa->sc->fd, &notify_rate); + ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rate_get(vc_ib->sc->fd, &notify_rate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**ibcom_sq_occupancy_notify_rate_get"); + "**MPID_nem_ib_com_sq_occupancy_notify_rate_get"); /* send RDMA-write-to buffer occupancy information */ /* embed SR occupancy information and remember the last one sent */ MPIDI_CH3_Pkt_t *ch3_hdr = (MPIDI_CH3_Pkt_t *) hdr; - if (MPID_nem_dcfa_diff32(*rsr_seq_num_tail, *rsr_seq_num_tail_last_sent) > notify_rate) { + if (MPID_nem_ib_diff32(*rsr_seq_num_tail, *rsr_seq_num_tail_last_sent) > notify_rate) { #if 1 /* debug, disabling piggy-back */ switch (ch3_hdr->type) { case MPIDI_CH3_PKT_EAGER_SEND: - pkt_netmod.subtype = MPIDI_NEM_DCFA_PKT_EAGER_SEND; + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_EAGER_SEND; goto common_tail; case MPIDI_CH3_PKT_PUT: - pkt_netmod.subtype = MPIDI_NEM_DCFA_PKT_PUT; + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_PUT; goto common_tail; case MPIDI_CH3_PKT_ACCUMULATE: - pkt_netmod.subtype = MPIDI_NEM_DCFA_PKT_ACCUMULATE; + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_ACCUMULATE; goto common_tail; case MPIDI_CH3_PKT_GET: - pkt_netmod.subtype = MPIDI_NEM_DCFA_PKT_GET; + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_GET; goto common_tail; case MPIDI_CH3_PKT_GET_RESP: - pkt_netmod.subtype = MPIDI_NEM_DCFA_PKT_GET_RESP; + pkt_netmod.subtype = MPIDI_NEM_IB_PKT_GET_RESP; common_tail: pkt_netmod.type = MPIDI_NEM_PKT_NETMOD; pkt_netmod.seq_num_tail = *rsr_seq_num_tail; *rsr_seq_num_tail_last_sent = *rsr_seq_num_tail; netmod_hdr = (void *) &pkt_netmod; - sz_netmod_hdr = sizeof(MPID_nem_dcfa_pkt_prefix_t); + sz_netmod_hdr = sizeof(MPID_nem_ib_pkt_prefix_t); break; default: netmod_hdr = NULL; @@ -220,7 +220,7 @@ static int MPID_nem_dcfa_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, /* packet handlers including MPIDI_CH3_PktHandler_EagerSend and MPID_nem_handle_pkt assume this */ hdr_sz = sizeof(MPIDI_CH3_Pkt_t); - /* send myrank as wr_id so that receiver can find vc using MPID_nem_dcfa_conns in poll */ + /* send myrank as wr_id so that receiver can find vc using MPID_nem_ib_conns in poll */ /* packet handler of MPIDI_CH3_PKT_EAGER_SEND uses sizeof(MPIDI_CH3_Pkt_t), so ignoring hdr_sz */ /* MPIDI_CH3_ReqHandler_GetSendRespComplete, drain_scq decrement it */ @@ -256,15 +256,15 @@ static int MPID_nem_dcfa_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, int copied; ibcom_errno = - ibcom_isend(vc_dcfa->sc->fd, (uint64_t) sreq, netmod_hdr, sz_netmod_hdr, hdr, hdr_sz, data, + MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, netmod_hdr, sz_netmod_hdr, hdr, hdr_sz, data, (int) data_sz, &copied); - MPIU_ERR_CHKFATALANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_isend"); - MPID_nem_dcfa_ncqe += 1; - //dprintf("isendcontig_core,ncqe=%d\n", MPID_nem_dcfa_ncqe); + MPIU_ERR_CHKFATALANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_isend"); + MPID_nem_ib_ncqe += 1; + //dprintf("isendcontig_core,ncqe=%d\n", MPID_nem_ib_ncqe); dprintf("isendcontig_core,isend,kind=%d,msg_type=%d,copied=%d\n", sreq->kind, msg_type, copied); /*suspicious lines,(if1,on,on,off,if0) works */ #if 0 -#define TLBPREF_AHEAD 20 //20 - int tlb_pref_ahd = 4096 * TLBPREF_AHEAD; +#define MPID_NEM_IB_TLBPREF_SEND 20 //20 + int tlb_pref_ahd = 4096 * MPID_NEM_IB_TLBPREF_SEND; __asm__ __volatile__ ("movq %0, %%rsi;" "movq 0(%%rsi), %%rax;"::"r"((uint64_t) data + tlb_pref_ahd):"%rsi", "%rax"); @@ -287,20 +287,20 @@ static int MPID_nem_dcfa_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, #endif #endif - MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "dcfa_send, fd=%d", vc_dcfa->sc->fd)); - vc_dcfa->pending_sends += 1; + MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "ib_send, fd=%d", vc_ib->sc->fd)); + vc_ib->pending_sends += 1; sreq->ch.vc = vc; /* used in poll */ /* calling drain_scq from progress_send deprives of chance - * for dcfa_poll to drain-sendq using ncqe + * for ib_poll to drain-sendq using ncqe * however transfers events to * (not to reply_seq_num because it's regulated by the rate) - * fire on dcfa_poll using nces + * fire on ib_poll using nces * (make SCQ full once, then put one command in sendq, * then send-->drain-scq to reduce CQE level under the threashold) * so we need to perform - * progress_send for all of VCs using nces in dcfa_poll - * (if we have drain-sendq in dcfa_poll, this isn't needed. */ + * progress_send for all of VCs using nces in ib_poll + * (if we have drain-sendq in ib_poll, this isn't needed. */ #if 0 /* debug,disabling fast-dec-cc when copied */ if (copied && !sreq->dev.OnDataAvail) { /* skip poll scq */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); @@ -336,74 +336,74 @@ static int MPID_nem_dcfa_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, } } else { - MPID_nem_dcfa_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ } #else - MPID_nem_dcfa_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-send */ #endif -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE - //dprintf("isendcontig,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE + //dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); int *notify_rstate; - ibcom_errno = ibcom_rdmabuf_occupancy_notify_rstate_get(vc_dcfa->sc->fd, &notify_rstate); + ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &notify_rstate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**ibcom_rdmabuf_occupancy_notify_rstate_get"); + "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get"); - dprintf("isendcontig,head=%d,tail=%d,hw=%d\n", vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail, IBCOM_RDMABUF_HIGH_WATER_MARK); + dprintf("isendcontig,head=%d,tail=%d,hw=%d\n", vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail, MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK); /* if the number of slots in RMDA-write-to buffer have hit the high water-mark */ - if (*notify_rstate == IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW && - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) > IBCOM_RDMABUF_HIGH_WATER_MARK) { - dprintf("changing notify_rstate,id=%d\n", vc_dcfa->ibcom->sseq_num); + if (*notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW && + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) > MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) { + dprintf("changing notify_rstate,id=%d\n", vc_ib->ibcom->sseq_num); /* remember remote notifying policy so that local can know when to change remote policy back to LW */ - *notify_rstate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW; + *notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW; /* change remote notifying policy of RDMA-write-to buf occupancy info */ - MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state(vc, - IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); + MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc, + MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); } - //dprintf("isendcontig_core,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + //dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_ISENDCONTIG_CORE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_ISENDCONTIG_CORE); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_iSendContig +#define FUNCNAME MPID_nem_ib_iSendContig #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, +int MPID_nem_ib_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_ISENDCONTIG); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_ISENDCONTIG); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ISENDCONTIG); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_ISENDCONTIG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); - MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "dcfa_iSendContig"); + MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "ib_iSendContig"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); -#ifdef DCFA_ONDEMAND - if (!vc_dcfa->is_connected) { - MPID_nem_dcfa_send_syn(vc); +#ifdef MPID_NEM_IB_ONDEMAND + if (!vc_ib->is_connected) { + MPID_nem_ib_send_syn(vc); } #endif #if 0 /* aggressively perform drain_scq */ /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_dcfa->ibcom->ncom >= /*IBCOM_MAX_SQ_CAPACITY */ IBCOM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_dcfa_ncqe >= /*IBCOM_MAX_CQ_CAPACITY */ IBCOM_MAX_CQ_HEIGHT_DRAIN) { + if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || + MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { //printf("isendcontig,kick drain_scq\n"); - ibcom_errno = MPID_nem_dcfa_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_drain_scq"); + ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); } #endif /* set it for drain_scq */ @@ -414,8 +414,8 @@ int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, /* Unexpected state MPIDI_VC_STATE_CLOSED in vc 0xf1fed0 (expecting MPIDI_VC_STATE_ACTIVE) * Assertion failed in file src/mpid/ch3/src/ch3u_handle_connection.c at line 326: vc->state == MPIDI_VC_STATE_ACTIVE */ if (vc->state == MPIDI_VC_STATE_ACTIVE && - MPID_nem_dcfa_tsc_poll - MPID_nem_dcfa_rdtsc() > MPID_NEM_DCFA_POLL_PERIOD_SEND) { - mpi_errno = MPID_nem_dcfa_poll(0); + MPID_nem_ib_tsc_poll - MPID_nem_ib_rdtsc() > MPID_NEM_IB_POLL_PERIOD_SEND) { + mpi_errno = MPID_nem_ib_poll(0); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -424,59 +424,59 @@ int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, int *lsr_seq_num_tail; /* sequence number of (largest) completed send command */ - ibcom_errno = ibcom_lsr_seq_num_tail_get(vc_dcfa->sc->fd, &lsr_seq_num_tail); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_lsr_seq_num_tail_get"); + ibcom_errno = MPID_nem_ib_com_lsr_seq_num_tail_get(vc_ib->sc->fd, &lsr_seq_num_tail); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lsr_seq_num_tail_get"); int lsr_seq_num_head; /* sequence number of (largest) in-flight send command */ - ibcom_errno = ibcom_sseq_num_get(vc_dcfa->sc->fd, &lsr_seq_num_head); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_sseq_num_get"); + ibcom_errno = MPID_nem_ib_com_sseq_num_get(vc_ib->sc->fd, &lsr_seq_num_head); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_sseq_num_get"); dprintf("isendcontig,%d->%d,type=%d,subtype=%d,data_sz=%ld,ldiff=%d(%d-%d),rdiff=%d(%d-%d)\n", - MPID_nem_dcfa_myrank, vc->pg_rank, ((MPIDI_CH3_Pkt_t *) hdr)->type, + MPID_nem_ib_myrank, vc->pg_rank, ((MPIDI_CH3_Pkt_t *) hdr)->type, ((MPID_nem_pkt_netmod_t *) hdr)->subtype, data_sz, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail), - vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - vc_dcfa->ibcom->rsr_seq_num_tail, vc_dcfa->ibcom->rsr_seq_num_tail_last_sent); + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail), + vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent); dprintf("isendcontig,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", - MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); /* if IB command overflow-queue is empty AND local IB command queue isn't full AND remote RDMA-write-to buf isn't getting overrun */ MPIDI_CH3_Pkt_t *ch3_hdr = (MPIDI_CH3_Pkt_t *) hdr; MPID_nem_pkt_netmod_t *netmod_hdr = (MPID_nem_pkt_netmod_t *) hdr; /* reserve one slot for control packet bringing sequence number * to avoid dead-lock */ int slack = ((ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || - netmod_hdr->subtype != MPIDI_NEM_DCFA_REQ_SEQ_NUM) && + netmod_hdr->subtype != MPIDI_NEM_IB_PKT_REQ_SEQ_NUM) && (ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || - netmod_hdr->subtype != MPIDI_NEM_DCFA_REPLY_SEQ_NUM) && + netmod_hdr->subtype != MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) && (ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || - netmod_hdr->subtype != MPIDI_NEM_DCFA_PKT_LMT_GET_DONE) && + netmod_hdr->subtype != MPIDI_NEM_IB_PKT_LMT_GET_DONE) && ch3_hdr->type != MPIDI_NEM_PKT_LMT_RTS && - ch3_hdr->type != MPIDI_NEM_PKT_LMT_CTS) ? IBCOM_AMT_SLACK : 0; + ch3_hdr->type != MPIDI_NEM_PKT_LMT_CTS) ? MPID_NEM_IB_COM_AMT_SLACK : 0; /* make control packet bringing sequence number go ahead of * queued packets to avoid dead-lock */ int goahead = - (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && netmod_hdr->subtype == MPIDI_NEM_DCFA_REQ_SEQ_NUM) + (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REQ_SEQ_NUM) || (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && - netmod_hdr->subtype == MPIDI_NEM_DCFA_REPLY_SEQ_NUM) || + netmod_hdr->subtype == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) || (ch3_hdr->type == MPIDI_NEM_PKT_NETMOD && - netmod_hdr->subtype == MPIDI_NEM_DCFA_PKT_LMT_GET_DONE) + netmod_hdr->subtype == MPIDI_NEM_IB_PKT_LMT_GET_DONE) ? 1 : 0; dprintf("isendcontig,slack=%d,goahead=%d\n", slack, goahead); if ( -#ifdef DCFA_ONDEMAND - vc_dcfa->is_connected && +#ifdef MPID_NEM_IB_ONDEMAND + vc_ib->is_connected && #endif - (goahead || MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)) && - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY - slack && - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY - slack && - MPID_nem_dcfa_diff32(lsr_seq_num_head, *lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG - slack) { + (goahead || MPID_nem_ib_sendq_empty(vc_ib->sendq)) && + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack && + MPID_nem_ib_diff32(lsr_seq_num_head, *lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { - mpi_errno = MPID_nem_dcfa_iSendContig_core(vc, sreq, hdr, hdr_sz, data, data_sz); + mpi_errno = MPID_nem_ib_iSendContig_core(vc, sreq, hdr, hdr_sz, data, data_sz); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -486,11 +486,11 @@ int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, /* enqueue command into send_queue */ dprintf("isendcontig,enqueuing,sendq=%d,ncom=%d,ncqe=%d,ldiff=%d(%d-%d),slack=%d\n", - (goahead || MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq)), - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY - slack, - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY - slack, - MPID_nem_dcfa_diff32(lsr_seq_num_head, *lsr_seq_num_tail), - vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail, slack); + (goahead || MPID_nem_ib_sendq_empty(vc_ib->sendq)), + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack, + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack, + MPID_nem_ib_diff32(lsr_seq_num_head, *lsr_seq_num_tail), + vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, slack); /* store required info. see MPIDI_CH3_iSendv in src/mpid/ch3/channels/nemesis/src/ch3_isendv.c */ sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr; @@ -525,9 +525,9 @@ int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, */ dprintf("isendcontig,enqueuing,type=%d,\n", ((MPIDI_CH3_Pkt_t *) hdr)->type); #if 0 - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_DCFA_REPLY_SEQ_NUM) { - printf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_dcfa_myrank, vc->pg_rank, - MPID_nem_dcfa_ncqe); + if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { + printf("enqueuing REPLY_SEQ_NUM\ %d->%d,%d\n", MPID_nem_ib_myrank, vc->pg_rank, + MPID_nem_ib_ncqe); } //if (((MPIDI_CH3_Pkt_t *)hdr)->type == MPIDI_CH3_PKT_ACCUMULATE) { //printf("enqueuing ACCUMULATE\n"); @@ -545,43 +545,43 @@ int MPID_nem_dcfa_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_PKT_LMT_DONE) { dprintf("isendcontig,enqueue,DONE\n"); } - if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_DCFA_REPLY_SEQ_NUM) { + if (((MPIDI_CH3_Pkt_t *) hdr)->type == MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) { dprintf("isendcontig,REPLY_SEQ_NUM,enqueue_at_head\n"); - MPID_nem_dcfa_sendq_enqueue_at_head(&vc_dcfa->sendq, sreq); + MPID_nem_ib_sendq_enqueue_at_head(&vc_ib->sendq, sreq); } else { - MPID_nem_dcfa_sendq_enqueue(&vc_dcfa->sendq, sreq); + MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, sreq); } /* we don't need to perform send_progress() here because * the events where RDMA-write-to buffer release is detected or release of IB command queue id detected happens - * only after dcfa_poll is called. it's different than the case where write(2) is used */ + * only after ib_poll is called. it's different than the case where write(2) is used */ } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_ISENDCONTIG); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_ISENDCONTIG); return mpi_errno; fn_fail: goto fn_exit; } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_iStartContigMsg +#define FUNCNAME MPID_nem_ib_iStartContigMsg #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, +int MPID_nem_ib_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr) { MPID_Request *sreq = NULL; int mpi_errno = MPI_SUCCESS; int ibcom_errno; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); int sseq_num; //uint64_t tscs, tsce; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_ISTARTCONTIGMSG); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_ISTARTCONTIGMSG); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_ISTARTCONTIGMSG); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_ISTARTCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); - MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "dcfa_iStartContigMsg"); + MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "ib_iStartContigMsg"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); /* FIXME: avoid creating a request when not queued */ @@ -593,39 +593,39 @@ int MPID_nem_dcfa_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr * released by PKT_GET_RESP, MPIDI_CH3I_RMAListComplete */ } - //tscs = MPID_nem_dcfa_rdtsc(); + //tscs = MPID_nem_ib_rdtsc(); sreq = MPID_Request_create(); MPIU_Assert(sreq != NULL); MPIU_Object_set_ref(sreq, 2); sreq->kind = MPID_REQUEST_SEND; sreq->dev.OnDataAvail = 0; - //tsce = MPID_nem_dcfa_rdtsc(); printf("rc,%ld\n", tsce - tscs); // 124.15 cycles + //tsce = MPID_nem_ib_rdtsc(); printf("rc,%ld\n", tsce - tscs); // 124.15 cycles #if 0 - ibcom_errno = ibcom_sseq_num_get(vc_dcfa->sc->fd, &sseq_num); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_sseq_num_get"); + ibcom_errno = MPID_nem_ib_com_sseq_num_get(vc_ib->sc->fd, &sseq_num); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_sseq_num_get"); if (hdr) { MPIDI_CH3_Pkt_t *pkt = (MPIDI_CH3_Pkt_t *) hdr; MPIDI_CH3_Pkt_close_t *close_pkt = &pkt->close; - dprintf("isend(istartcontig),%d->%d,seq_num=%d,type=%d,ack=%d\n", MPID_nem_dcfa_myrank, + dprintf("isend(istartcontig),%d->%d,seq_num=%d,type=%d,ack=%d\n", MPID_nem_ib_myrank, vc->pg_rank, sseq_num, close_pkt->type, close_pkt->ack); } else { - dprintf("isend(istartcontig),%d->%d,seq_num=%d\n", MPID_nem_dcfa_myrank, vc->pg_rank, + dprintf("isend(istartcontig),%d->%d,seq_num=%d\n", MPID_nem_ib_myrank, vc->pg_rank, sseq_num); } #endif - mpi_errno = MPID_nem_dcfa_iSendContig(vc, sreq, hdr, hdr_sz, data, data_sz); + mpi_errno = MPID_nem_ib_iSendContig(vc, sreq, hdr, hdr_sz, data, data_sz); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } fn_exit: *sreq_ptr = sreq; - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_ISTARTCONTIGMSG); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_ISTARTCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; @@ -633,20 +633,20 @@ int MPID_nem_dcfa_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_SendNoncontig_core +#define FUNCNAME MPID_nem_ib_SendNoncontig_core #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -static int MPID_nem_dcfa_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, +static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; MPIDI_msg_sz_t last; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); int sseq_num; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_SENDNONCONTIG_CORE); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_SENDNONCONTIG_CORE); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG_CORE); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG_CORE); MPIU_Assert(sreq->dev.segment_first == 0); last = sreq->dev.segment_size; /* segment_size is byte offset */ @@ -664,20 +664,20 @@ static int MPID_nem_dcfa_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq MPIR_Request_add_ref(sreq); } - ibcom_errno = ibcom_sseq_num_get(vc_dcfa->sc->fd, &sseq_num); - MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**ibcom_sseq_num_get"); + ibcom_errno = MPID_nem_ib_com_sseq_num_get(vc_ib->sc->fd, &sseq_num); + MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_sseq_num_get"); int copied; - dprintf("sendnoncontig_core,isend,%d->%d,seq_num=%d\n", MPID_nem_dcfa_myrank, vc->pg_rank, + dprintf("sendnoncontig_core,isend,%d->%d,seq_num=%d\n", MPID_nem_ib_myrank, vc->pg_rank, sseq_num); ibcom_errno = - ibcom_isend(vc_dcfa->sc->fd, (uint64_t) sreq, NULL, 0, hdr, sizeof(MPIDI_CH3_Pkt_t), + MPID_nem_ib_com_isend(vc_ib->sc->fd, (uint64_t) sreq, NULL, 0, hdr, sizeof(MPIDI_CH3_Pkt_t), (void *) REQ_FIELD(sreq, lmt_pack_buf), (int) last, &copied); - MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**ibcom_isend"); - MPID_nem_dcfa_ncqe += 1; - dprintf("sendnoncontig_core,ncqe=%d\n", MPID_nem_dcfa_ncqe); + MPIU_ERR_CHKANDJUMP(ibcom_errno != 0, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_isend"); + MPID_nem_ib_ncqe += 1; + dprintf("sendnoncontig_core,ncqe=%d\n", MPID_nem_ib_ncqe); - vc_dcfa->pending_sends += 1; + vc_ib->pending_sends += 1; sreq->ch.vc = vc; /* used in poll */ #if 0 /* see contig */ @@ -715,38 +715,38 @@ static int MPID_nem_dcfa_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq } } else { - MPID_nem_dcfa_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ } #else - MPID_nem_dcfa_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ + MPID_nem_ib_ncqe_nces += 1; /* it has different meaning, counting non-copied eager-short */ #endif -#ifndef DISABLE_VAR_OCC_NOTIFY_RATE +#ifndef MPID_NEM_IB_DISABLE_VAR_OCC_NOTIFY_RATE #if 1 - //dprintf("isendcontig,old rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + //dprintf("isendcontig,old rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); int *notify_rstate; - ibcom_errno = ibcom_rdmabuf_occupancy_notify_rstate_get(vc_dcfa->sc->fd, &notify_rstate); + ibcom_errno = MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get(vc_ib->sc->fd, &notify_rstate); MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**ibcom_rdmabuf_occupancy_notify_rstate_get"); + "**MPID_nem_ib_com_rdmabuf_occupancy_notify_rstate_get"); - dprintf("isendcontig,head=%d,tail=%d,hw=%d\n", vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail, IBCOM_RDMABUF_HIGH_WATER_MARK); + dprintf("isendcontig,head=%d,tail=%d,hw=%d\n", vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail, MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK); /* if the number of slots in RMDA-write-to buffer have hit the high water-mark */ - if (*notify_rstate == IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW && - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) > IBCOM_RDMABUF_HIGH_WATER_MARK) { - dprintf("changing notify_rstate,id=%d\n", vc_dcfa->ibcom->sseq_num); + if (*notify_rstate == MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW && + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) > MPID_NEM_IB_COM_RDMABUF_HIGH_WATER_MARK) { + dprintf("changing notify_rstate,id=%d\n", vc_ib->ibcom->sseq_num); /* remember remote notifying policy so that local can know when to change remote policy back to LW */ - *notify_rstate = IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW; + *notify_rstate = MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW; /* change remote notifying policy of RDMA-write-to buf occupancy info */ - MPID_nem_dcfa_send_change_rdmabuf_occupancy_notify_state(vc, - IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); + MPID_nem_ib_send_change_rdmabuf_occupancy_notify_state(vc, + MPID_NEM_IB_COM_RDMABUF_OCCUPANCY_NOTIFY_STATE_HW); } - //dprintf("isendcontig_core,new rstate=%d\n", vc_dcfa->ibcom->rdmabuf_occupancy_notify_rstate); + //dprintf("isendcontig_core,new rstate=%d\n", vc_ib->ibcom->rdmabuf_occupancy_notify_rstate); #endif #endif fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_SENDNONCONTIG_CORE); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG_CORE); return mpi_errno; fn_fail: goto fn_exit; @@ -754,34 +754,34 @@ static int MPID_nem_dcfa_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_SendNoncontig +#define FUNCNAME MPID_nem_ib_SendNoncontig #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, +int MPID_nem_ib_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; MPIDI_msg_sz_t last; - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); + MPID_nem_ib_vc_area *vc_ib = VC_IB(vc); int sseq_num; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_SENDNONCONTIG); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_SENDNONCONTIG); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); - MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "dcfa_SendNoncontig"); + MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "ib_SendNoncontig"); - dprintf("sendnoncontig,%d->%d,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_myrank, - vc->pg_rank, MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + dprintf("sendnoncontig,%d->%d,sendq_empty=%d,ncom=%d,rdmabuf_occ=%d\n", MPID_nem_ib_myrank, + vc->pg_rank, MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); #if 0 /* aggressively perform drain_scq */ /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_dcfa->ibcom->ncom >= /*IBCOM_MAX_SQ_CAPACITY */ IBCOM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_dcfa_ncqe >= /*IBCOM_MAX_CQ_CAPACITY */ IBCOM_MAX_CQ_HEIGHT_DRAIN) { + if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || + MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { //printf("isendcontig,kick drain_scq\n"); - ibcom_errno = MPID_nem_dcfa_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_drain_scq"); + ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_drain_scq"); } #endif /* set it for drain_scq */ @@ -789,14 +789,14 @@ int MPID_nem_dcfa_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, /* if IB command overflow-queue is empty AND local IB command queue isn't full AND remote RDMA-write-to buf isn't getting overrun */ /* set it for drain_scq */ - int slack = IBCOM_AMT_SLACK; /* slack for control packet bringing sequence number */ - if (MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq) && - vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY - slack && - MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY - slack && - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG - slack) { - - mpi_errno = MPID_nem_dcfa_SendNoncontig_core(vc, sreq, hdr, hdr_sz); + int slack = MPID_NEM_IB_COM_AMT_SLACK; /* slack for control packet bringing sequence number */ + if (MPID_nem_ib_sendq_empty(vc_ib->sendq) && + vc_ib->ibcom->ncom < MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack && + MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack && + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) < MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { + + mpi_errno = MPID_nem_ib_SendNoncontig_core(vc, sreq, hdr, hdr_sz); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } @@ -816,11 +816,11 @@ int MPID_nem_dcfa_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, sreq->ch.noncontig = TRUE; sreq->ch.vc = vc; - MPID_nem_dcfa_sendq_enqueue(&vc_dcfa->sendq, sreq); + MPID_nem_ib_sendq_enqueue(&vc_ib->sendq, sreq); } fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_SENDNONCONTIG); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_SENDNONCONTIG); return mpi_errno; fn_fail: goto fn_exit; @@ -828,10 +828,10 @@ int MPID_nem_dcfa_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, /* see MPIDI_CH3I_Shm_send_progress (in src/mpid/ch3/channels/nemesis/src/ch3_progress.c) */ #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_send_progress +#define FUNCNAME MPID_nem_ib_send_progress #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) +int MPID_nem_ib_send_progress(MPID_nem_ib_vc_area * vc_ib) { int mpi_errno = MPI_SUCCESS; int ibcom_errno; @@ -841,8 +841,8 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) int again = 0; int msg_type; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_SEND_PROGRESS); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_SEND_PROGRESS); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_SEND_PROGRESS); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_SEND_PROGRESS); //dprintf("send_progress,enter\n"); @@ -852,19 +852,19 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) } entered_send_progress = 1; - sreq = MPID_nem_dcfa_sendq_head(vc_dcfa->sendq); + sreq = MPID_nem_ib_sendq_head(vc_ib->sendq); if (sreq) { prev_sreq = NULL; do { #if 0 /* aggressively perform drain_scq */ /* try to clear the road blocks, i.e. ncom, ncqe */ - if (vc_dcfa->ibcom->ncom >= /*IBCOM_MAX_SQ_CAPACITY */ IBCOM_MAX_SQ_HEIGHT_DRAIN || - MPID_nem_dcfa_ncqe >= /*IBCOM_MAX_CQ_CAPACITY */ IBCOM_MAX_CQ_HEIGHT_DRAIN) { + if (vc_ib->ibcom->ncom >= /*MPID_NEM_IB_COM_MAX_SQ_CAPACITY */ MPID_NEM_IB_COM_MAX_SQ_HEIGHT_DRAIN || + MPID_nem_ib_ncqe >= /*MPID_NEM_IB_COM_MAX_CQ_CAPACITY */ MPID_NEM_IB_COM_MAX_CQ_HEIGHT_DRAIN) { dprintf("send_progress,kick drain_scq\n"); - ibcom_errno = MPID_nem_dcfa_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ + ibcom_errno = MPID_nem_ib_drain_scq(1); /* set 1st arg to one which means asking it for not calling send_progress because it recursively call isendcontig_core */ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq"); + "**MPID_nem_ib_drain_scq"); } #endif msg_type = MPIDI_Request_get_msg_type(sreq); @@ -874,43 +874,43 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) (MPID_nem_pkt_netmod_t *) sreq->dev.iov[0].MPID_IOV_BUF; int slack = (msg_type == MPIDI_REQUEST_EAGER_MSG) ? /* guard from RDMA-read or RDMA-write */ (((ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || - netmod_hdr->subtype != MPIDI_NEM_DCFA_REQ_SEQ_NUM) && + netmod_hdr->subtype != MPIDI_NEM_IB_PKT_REQ_SEQ_NUM) && (ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || - netmod_hdr->subtype != MPIDI_NEM_DCFA_REPLY_SEQ_NUM) && + netmod_hdr->subtype != MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM) && (ch3_hdr->type != MPIDI_NEM_PKT_NETMOD || - netmod_hdr->subtype != MPIDI_NEM_DCFA_PKT_LMT_GET_DONE) && + netmod_hdr->subtype != MPIDI_NEM_IB_PKT_LMT_GET_DONE) && ch3_hdr->type != MPIDI_NEM_PKT_LMT_RTS && - ch3_hdr->type != MPIDI_NEM_PKT_LMT_CTS) ? IBCOM_AMT_SLACK : 0) : IBCOM_AMT_SLACK; - if (vc_dcfa->ibcom->ncom >= IBCOM_MAX_SQ_CAPACITY - slack || - MPID_nem_dcfa_ncqe >= IBCOM_MAX_CQ_CAPACITY - slack || - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail) >= - IBCOM_RDMABUF_NSEG - slack) { + ch3_hdr->type != MPIDI_NEM_PKT_LMT_CTS) ? MPID_NEM_IB_COM_AMT_SLACK : 0) : MPID_NEM_IB_COM_AMT_SLACK; + if (vc_ib->ibcom->ncom >= MPID_NEM_IB_COM_MAX_SQ_CAPACITY - slack || + MPID_nem_ib_ncqe >= MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack || + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail) >= + MPID_NEM_IB_COM_RDMABUF_NSEG - slack) { break; } - if (vc_dcfa != MPID_nem_dcfa_debug_current_vc_dcfa) { - dprintf("send_progress,vc_dcfa != MPID_nem_dcfa_debug_current_vc_dcfa\n"); + if (vc_ib != MPID_nem_ib_debug_current_vc_ib) { + dprintf("send_progress,vc_ib != MPID_nem_ib_debug_current_vc_ib\n"); } dprintf("send_progress,kind=%d,msg_type=%d\n", sreq->kind, msg_type); if (msg_type == MPIDI_REQUEST_EAGER_MSG) { dprintf("send_progress,type=%d\n", ch3_hdr->type); } dprintf("send_progress,%d->%d,rdiff=%d(%d-%d),ldiff=%d(%d-%d),slack=%d\n", - MPID_nem_dcfa_myrank, sreq->ch.vc->pg_rank, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->rsr_seq_num_tail, - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent), - vc_dcfa->ibcom->rsr_seq_num_tail, vc_dcfa->ibcom->rsr_seq_num_tail_last_sent, - MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, - vc_dcfa->ibcom->lsr_seq_num_tail), - vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail, slack); + MPID_nem_ib_myrank, sreq->ch.vc->pg_rank, + MPID_nem_ib_diff32(vc_ib->ibcom->rsr_seq_num_tail, + vc_ib->ibcom->rsr_seq_num_tail_last_sent), + vc_ib->ibcom->rsr_seq_num_tail, vc_ib->ibcom->rsr_seq_num_tail_last_sent, + MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, + vc_ib->ibcom->lsr_seq_num_tail), + vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail, slack); if (sreq->kind == MPID_REQUEST_SEND && msg_type == MPIDI_REQUEST_EAGER_MSG) { if (!sreq->ch.noncontig) { dprintf - ("send_progress,contig,type=%d,sseq_num=%d,MPIDI_NEM_PKT_LMT_RTS=%d,MPIDI_NEM_DCFA_PKT_LMT_GET_DONE=%d\n", - ch3_hdr->type, vc_dcfa->ibcom->sseq_num, MPIDI_NEM_PKT_LMT_RTS, - MPIDI_NEM_DCFA_PKT_LMT_GET_DONE); + ("send_progress,contig,type=%d,sseq_num=%d,MPIDI_NEM_PKT_LMT_RTS=%d,MPIDI_NEM_IB_PKT_LMT_GET_DONE=%d\n", + ch3_hdr->type, vc_ib->ibcom->sseq_num, MPIDI_NEM_PKT_LMT_RTS, + MPIDI_NEM_IB_PKT_LMT_GET_DONE); if (sreq->dev.iov[1].MPID_IOV_LEN > 0) { dprintf ("send_progress,send,contig,sreq->dev.iov[1].MPID_IOV_BUF)=%p,*(sreq->dev.iov[1].MPID_IOV_BUF)=%08x\n", @@ -923,28 +923,28 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) /* send current rsr_seq_num_tail because message from target to initiator * might have happened while being queued */ case MPIDI_NEM_PKT_LMT_RTS:{ - MPID_nem_dcfa_lmt_cookie_t *s_cookie_buf = - (MPID_nem_dcfa_lmt_cookie_t *) sreq->dev.iov[1].MPID_IOV_BUF; + MPID_nem_ib_lmt_cookie_t *s_cookie_buf = + (MPID_nem_ib_lmt_cookie_t *) sreq->dev.iov[1].MPID_IOV_BUF; dprintf("send_progress,MPIDI_NEM_PKT_LMT_RTS,rsr_seq_num_tail=%d\n", - vc_dcfa->ibcom->rsr_seq_num_tail); + vc_ib->ibcom->rsr_seq_num_tail); /* embed RDMA-write-to buffer occupancy information */ - s_cookie_buf->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; + s_cookie_buf->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; /* remember the last one sent */ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = - vc_dcfa->ibcom->rsr_seq_num_tail; + vc_ib->ibcom->rsr_seq_num_tail_last_sent = + vc_ib->ibcom->rsr_seq_num_tail; break; } case MPIDI_NEM_PKT_LMT_CTS:{ - MPID_nem_dcfa_lmt_cookie_t *s_cookie_buf = - (MPID_nem_dcfa_lmt_cookie_t *) sreq->dev.iov[1].MPID_IOV_BUF; + MPID_nem_ib_lmt_cookie_t *s_cookie_buf = + (MPID_nem_ib_lmt_cookie_t *) sreq->dev.iov[1].MPID_IOV_BUF; dprintf("send_progress,MPIDI_NEM_PKT_LMT_CTS,rsr_seq_num_tail=%d\n", - vc_dcfa->ibcom->rsr_seq_num_tail); + vc_ib->ibcom->rsr_seq_num_tail); /* embed RDMA-write-to buffer occupancy information */ - s_cookie_buf->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; + s_cookie_buf->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; /* remember the last one sent */ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = - vc_dcfa->ibcom->rsr_seq_num_tail; + vc_ib->ibcom->rsr_seq_num_tail_last_sent = + vc_ib->ibcom->rsr_seq_num_tail; break; } @@ -955,32 +955,32 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) switch (netmod_hdr->subtype) { /* send current rsr_seq_num_tail because message from target to initiator * might have happened while being queued */ - case MPIDI_NEM_DCFA_PKT_LMT_GET_DONE:{ - MPID_nem_dcfa_pkt_lmt_get_done_t *_done_pkt = - (MPID_nem_dcfa_pkt_lmt_get_done_t *) sreq->dev.iov[0]. + case MPIDI_NEM_IB_PKT_LMT_GET_DONE:{ + MPID_nem_ib_pkt_lmt_get_done_t *_done_pkt = + (MPID_nem_ib_pkt_lmt_get_done_t *) sreq->dev.iov[0]. MPID_IOV_BUF; dprintf - ("send_progress,MPIDI_NEM_DCFA_PKT_LMT_GET_DONE,rsr_seq_num_tail=%d\n", - vc_dcfa->ibcom->rsr_seq_num_tail); + ("send_progress,MPIDI_NEM_IB_PKT_LMT_GET_DONE,rsr_seq_num_tail=%d\n", + vc_ib->ibcom->rsr_seq_num_tail); /* embed SR occupancy information */ - _done_pkt->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; + _done_pkt->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; /* remember the last one sent */ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = - vc_dcfa->ibcom->rsr_seq_num_tail; + vc_ib->ibcom->rsr_seq_num_tail_last_sent = + vc_ib->ibcom->rsr_seq_num_tail; break; } - case MPIDI_NEM_DCFA_REPLY_SEQ_NUM:{ - MPID_nem_dcfa_pkt_reply_seq_num_t *_pkt = - (MPID_nem_dcfa_pkt_reply_seq_num_t *) sreq->dev.iov[0]. + case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:{ + MPID_nem_ib_pkt_reply_seq_num_t *_pkt = + (MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.iov[0]. MPID_IOV_BUF; dprintf - ("send_progress,MPIDI_NEM_DCFA_REPLY_SEQ_NUM,rsr_seq_num_tail=%d\n", - vc_dcfa->ibcom->rsr_seq_num_tail); + ("send_progress,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM,rsr_seq_num_tail=%d\n", + vc_ib->ibcom->rsr_seq_num_tail); /* embed SR occupancy information */ - _pkt->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; + _pkt->seq_num_tail = vc_ib->ibcom->rsr_seq_num_tail; /* remember the last one sent */ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = - vc_dcfa->ibcom->rsr_seq_num_tail; + vc_ib->ibcom->rsr_seq_num_tail_last_sent = + vc_ib->ibcom->rsr_seq_num_tail; break; } @@ -990,7 +990,7 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) mpi_errno = - MPID_nem_dcfa_iSendContig_core(sreq->ch.vc, sreq, + MPID_nem_ib_iSendContig_core(sreq->ch.vc, sreq, sreq->dev.iov[0].MPID_IOV_BUF, sreq->dev.iov[0].MPID_IOV_LEN, sreq->dev.iov[1].MPID_IOV_BUF, @@ -1002,7 +1002,7 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) else { dprintf("send_progress,send,noncontig\n"); mpi_errno = - MPID_nem_dcfa_SendNoncontig_core(sreq->ch.vc, sreq, + MPID_nem_ib_SendNoncontig_core(sreq->ch.vc, sreq, sreq->dev.iov[0].MPID_IOV_BUF, sreq->dev.iov[0].MPID_IOV_LEN); if (mpi_errno) { @@ -1013,9 +1013,9 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) else if (sreq->kind == MPID_REQUEST_RECV && msg_type == MPIDI_REQUEST_RNDV_MSG) { dprintf("send_progress,kick lmt_start_recv_core,prev=%p,next=%p\n", prev_sreq, - MPID_nem_dcfa_sendq_next(sreq)); + MPID_nem_ib_sendq_next(sreq)); mpi_errno = - MPID_nem_dcfa_lmt_start_recv_core(sreq, REQ_FIELD(sreq, lmt_raddr), + MPID_nem_ib_lmt_start_recv_core(sreq, REQ_FIELD(sreq, lmt_raddr), REQ_FIELD(sreq, lmt_rkey), REQ_FIELD(sreq, lmt_write_to_buf)); if (mpi_errno) { @@ -1033,80 +1033,80 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) /* unlink sreq */ if (prev_sreq != NULL) { - MPID_nem_dcfa_sendq_next(prev_sreq) = MPID_nem_dcfa_sendq_next(sreq); + MPID_nem_ib_sendq_next(prev_sreq) = MPID_nem_ib_sendq_next(sreq); } else { - MPID_nem_dcfa_sendq_head(vc_dcfa->sendq) = MPID_nem_dcfa_sendq_next(sreq); + MPID_nem_ib_sendq_head(vc_ib->sendq) = MPID_nem_ib_sendq_next(sreq); } - if (MPID_nem_dcfa_sendq_next(sreq) == NULL) { - vc_dcfa->sendq.tail = prev_sreq; + if (MPID_nem_ib_sendq_next(sreq) == NULL) { + vc_ib->sendq.tail = prev_sreq; } /* save sreq->dev.next (and sreq) because decrementing reference-counter might free sreq */ MPID_Request *tmp_sreq = sreq; - sreq = MPID_nem_dcfa_sendq_next(sreq); + sreq = MPID_nem_ib_sendq_next(sreq); goto next_unlinked; //next: prev_sreq = sreq; - sreq = MPID_nem_dcfa_sendq_next(sreq); + sreq = MPID_nem_ib_sendq_next(sreq); next_unlinked:; } while (sreq); } - //dprintf("send_progress,exit,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); + //dprintf("send_progress,exit,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_ib_sendq_empty(vc_ib->sendq), vc_ib->ibcom->ncom, MPID_nem_ib_ncqe, MPID_nem_ib_diff32(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail)); fn_exit: entered_send_progress = 0; - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_SEND_PROGRESS); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_SEND_PROGRESS); return mpi_errno; fn_fail: goto fn_exit; } -#ifdef DCFA_ONDEMAND -int MPID_nem_dcfa_cm_send_core(int rank, MPID_nem_dcfa_cm_cmd_t * cmd) +#ifdef MPID_NEM_IB_ONDEMAND +int MPID_nem_ib_cm_send_core(int rank, MPID_nem_ib_cm_cmd_t * cmd) { - IbCom *ibcom_scratch_pad; - ibcom_errno = ibcom_obtain_pointer(MPID_nem_dcfa_scratch_pad_fds[rank], &ibcom_scratch_pad); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_obtain_pointer"); + MPID_nem_ib_com *ibcom_scratch_pad; + ibcom_errno = MPID_nem_ib_com_obtain_pointer(MPID_nem_ib_scratch_pad_fds[rank], &ibcom_scratch_pad); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer"); - if (MPID_nem_dcfa_ncqe_scratch_pad >= IBCOM_MAX_SQ_CAPACITY || - ibcom_scratch_pad->ncom_scratch_pad >= IBCOM_MAX_CQ_CAPACITY) { - mpi_errno = MPID_nem_dcfa_drain_scq_scratch_pad(); + if (MPID_nem_ib_ncqe_scratch_pad >= MPID_NEM_IB_COM_MAX_SQ_CAPACITY || + ibcom_scratch_pad->ncom_scratch_pad >= MPID_NEM_IB_COM_MAX_CQ_CAPACITY) { + mpi_errno = MPID_nem_ib_drain_scq_scratch_pad(); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, - "**MPID_nem_dcfa_drain_scq_scratch_pad"); + "**MPID_nem_ib_drain_scq_scratch_pad"); } ibcom_errno = - ibcom_put_scratch_pad(MPID_nem_dcfa_scratch_pad_fds[rank], (uint64_t) ibcom_scratch_pad, - sizeof(uint32_t), sizeof(MPID_nem_dcfa_cm_cmd_t), (void *) cmd); - MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_put_scratch_pad"); - MPID_nem_dcfa_ncqe_scratch_pad += 1; + MPID_nem_ib_com_put_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) ibcom_scratch_pad, + sizeof(uint32_t), sizeof(MPID_nem_ib_cm_cmd_t), (void *) cmd); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_put_scratch_pad"); + MPID_nem_ib_ncqe_scratch_pad += 1; /* atomic write to doorbell */ } #undef FUNCNAME -#define FUNCNAME MPID_nem_dcfa_cm_connect +#define FUNCNAME MPID_nem_ib_cm_connect #undef FCNAME #define FCNAME MPIU_QUOTE(FUNCNAME) -int MPID_nem_dcfa_cm_connect(MPIDI_VC_t * vc) +int MPID_nem_ib_cm_connect(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; int val; - MPID_nem_dcfa_cm_cmd_t cmd; + MPID_nem_ib_cm_cmd_t cmd; - MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_CM_CONNECT); - MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_CM_CONNECT); + MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_CM_CONNECT); + MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_CM_CONNECT); dprintf("connect,enter\n"); - cmd.type = MPID_NEM_DCFA_CM_SYN; - mpi_errno = MPID_nem_dcfa_cm_send_core(rank, &cmd); - MPIU_ERR_CHKANDJUMP(mp_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_cm_put"); + cmd.type = MPID_NEM_IB_CM_SYN; + mpi_errno = MPID_nem_ib_cm_send_core(rank, &cmd); + MPIU_ERR_CHKANDJUMP(mp_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_put"); fn_exit: - MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_CM_CONNECT); + MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_CM_CONNECT); return mpi_errno; fn_fail: goto fn_exit; diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 similarity index 70% rename from src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 rename to src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 index b7a5d08..5026002 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 @@ -4,18 +4,18 @@ dnl MPICH_SUBCFG_AFTER=src/mpid/ch3/channels/nemesis AC_DEFUN([PAC_SUBCFG_PREREQ_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_CH3_NEMESIS],[ for net in $nemesis_networks ; do - AS_CASE([$net],[dcfa],[build_nemesis_netmod_dcfa=yes]) + AS_CASE([$net],[ib],[build_nemesis_netmod_ib=yes]) done ]) - AM_CONDITIONAL([BUILD_NEMESIS_NETMOD_DCFA],[test "X$build_nemesis_netmod_dcfa" = "Xyes"]) + AM_CONDITIONAL([BUILD_NEMESIS_NETMOD_IB],[test "X$build_nemesis_netmod_ib" = "Xyes"]) # check if getpagesize is available AC_CHECK_FUNCS(getpagesize) ])dnl AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ -AM_COND_IF([BUILD_NEMESIS_NETMOD_DCFA],[ - AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:dcfa]) +AM_COND_IF([BUILD_NEMESIS_NETMOD_IB],[ + AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:ib]) PAC_CHECK_HEADER_LIB(dcfa.h,dcfa,ibv_open_device,dcfa_found=yes,dcfa_found=no) if test "${dcfa_found}" = "yes" ; then @@ -29,11 +29,11 @@ AM_COND_IF([BUILD_NEMESIS_NETMOD_DCFA],[ fi fi - AC_DEFINE([MPID_NEM_DCFA_VERSION], ["0.9.0"], [Version of netmod/DCFA]) - AC_DEFINE([MPID_NEM_DCFA_RELEASE_DATE], ["2013-11-18"], [Release date of netmod/DCFA]) - AC_DEFINE([ENABLE_COMM_OVERRIDES], 1, [define to add per-vc function pointers to override send and recv functions, registered in dcfa_init.c]) + AC_DEFINE([MPID_NEM_IB_VERSION], ["0.9.0"], [Version of the IB netmod]) + AC_DEFINE([MPID_NEM_IB_RELEASE_DATE], ["2013-11-18"], [Release date of the IB netmod]) + AC_DEFINE([ENABLE_COMM_OVERRIDES], 1, [define to add per-vc function pointers to override send and recv functions, registered in ib_init.c]) # AC_DEFINE([ENABLE_RNDV_WAIT_TIMER], 1, [make MPI_Wtime returns wait time. Wait time is elapsed time from MPIDI_CH3_Progress_start to MPIDI_CH3_Progress_end]) -])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_DCFA,...) +])dnl end AM_COND_IF(BUILD_NEMESIS_NETMOD_IB,...) ])dnl end _BODY [#] end of __file__ diff --git a/src/mpid/ch3/channels/nemesis/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/subconfigure.m4 index 2ebaa97..9c69219 100644 --- a/src/mpid/ch3/channels/nemesis/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/subconfigure.m4 @@ -94,7 +94,7 @@ AC_CHECK_FUNCS(signal) # Check for netmod relevant headers and libraries PAC_SET_HEADER_LIB_PATH(mx) PAC_SET_HEADER_LIB_PATH(scif) -PAC_SET_HEADER_LIB_PATH(dcfa) +PAC_SET_HEADER_LIB_PATH(ib) nemesis_nets_dirs="" nemesis_nets_strings="" http://git.mpich.org/mpich.git/commitdiff/b9f5aa8fd45accaeefe9c4e2ba2ad6852… commit b9f5aa8fd45accaeefe9c4e2ba2ad6852644bad0 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Fri Dec 13 11:14:12 2013 +0900 Remove macro for receiver-initiate lmt Remove PUT_DONE macro. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h index 2ee3bbb..9922d0c 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h @@ -16,7 +16,6 @@ #include <sys/mman.h> #define LMT_GET_CQE /* detect RDMA completion by CQE */ -//#define LMT_PUT_DONE #define DISABLE_VAR_OCC_NOTIFY_RATE /* lmt-put: (1) receiver sends cts to sender (2) sender RDMA-write to receiver diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c index a0765ec..873c671 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c @@ -946,11 +946,11 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) #endif // lazy feching of completion queue entry because it causes cache-miss -#if !defined (LMT_PUT_DONE) && defined (LMT_GET_CQE) +#if defined (LMT_GET_CQE) if (MPID_nem_dcfa_ncqe_to_drain > 0 || MPID_nem_dcfa_ncqe_nces > 0 || MPID_nem_dcfa_ncqe >= IBCOM_MAX_CQ_HEIGHT_DRAIN || ncom_almost_full) #endif -#if !defined (LMT_PUT_DONE) && !defined (LMT_GET_CQE) +#if !defined (LMT_GET_CQE) if (/*(in_blocking_poll && result == 0) || */ MPID_nem_dcfa_ncqe_nces > 0 || MPID_nem_dcfa_ncqe >= IBCOM_MAX_CQ_HEIGHT_DRAIN || ncom_almost_full) #endif http://git.mpich.org/mpich.git/commitdiff/44f906a47c12b0c1e037fd3374a79606a… commit 44f906a47c12b0c1e037fd3374a79606aca002e6 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Mon Dec 9 09:48:38 2013 +0900 Correct debug messages Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c index 302808d..a0765ec 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c @@ -1290,7 +1290,7 @@ int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_EAGERSEND); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_DCFA_PKTHANDLER_EAGERSEND); - printf("dcfa_pkthandler_eagersend,tag=%d\n", ch3_pkt->match.parts.tag); + dprintf("dcfa_pkthandler_eagersend,tag=%d\n", ch3_pkt->match.parts.tag); /* Check the assumption on sizeof(MPIDI_CH3_Pkt_t). * It is utilized to point the payload location in MPIDI_CH3_PktHandler_EagerSend @@ -1341,12 +1341,12 @@ int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, #if 0 mpi_errno = MPID_nem_handle_pkt(vc, (char *) pkt_parent_class, *buflen); #else - printf("dcfa_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); + dprintf("dcfa_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen); MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_dcfa_pkt_prefix_t); mpi_errno = MPIDI_CH3_PktHandler_EagerSend(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp); - printf("dcfa_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen); + dprintf("dcfa_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen); *buflen = ch3_buflen + sizeof(MPID_nem_dcfa_pkt_prefix_t); - printf("dcfa_poll.c,after addition,buflen=%ld\n", *buflen); + dprintf("dcfa_poll.c,after addition,buflen=%ld\n", *buflen); #endif if (mpi_errno) { MPIU_ERR_POP(mpi_errno); diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c index 42fe244..bfd23d9 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c @@ -199,9 +199,9 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) //dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next); if (e->addr <= addr_aligned && (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) { - dprintf - ("ibcom_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n", - e->addr, e->len, e->mr->addr, e->mr->length, addr, len); + //dprintf + //("ibcom_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n", + //e->addr, e->len, e->mr->addr, e->mr->length, addr, len); goto hit; } } @@ -214,8 +214,8 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) (struct ibcom_reg_mr_cache_entry_t *) e->lru_prev; ibcom_reg_mr_unlink((struct ibcom_reg_mr_listnode_t *) victim); - dprintf("ibcom_reg_mr,evict,entry addr=%p,len=%d,mr addr=%p,len=%ld\n", e->addr, e->len, - e->mr->addr, e->mr->length); + //dprintf("ibcom_reg_mr,evict,entry addr=%p,len=%d,mr addr=%p,len=%ld\n", e->addr, e->len, + //e->mr->addr, e->mr->length); ibcom_errno = ibcom_dereg_mr(victim->mr); if (ibcom_errno) { printf("mrcache,ibcom_dereg_mr\n"); @@ -238,8 +238,8 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) e->addr = addr_aligned; e->len = len_aligned; - dprintf("ibcom_reg_mr_fetch,fill,e=%p,key=%d,mr=%p,mr addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, - key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); + //dprintf("ibcom_reg_mr_fetch,fill,e=%p,key=%d,mr=%p,mr addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, + //key, e->mr, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); /* register to cache */ ibcom_reg_mr_insert(&ibcom_reg_mr_cache[key], (struct ibcom_reg_mr_listnode_t *) e); @@ -259,8 +259,8 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) ibcom_reg_mr_insert(&ibcom_reg_mr_cache[key], (struct ibcom_reg_mr_listnode_t *) e); } #endif - dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, - key, e->mr, e->refc, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); + //dprintf("[MrCache] reuse e=%p,key=%d,mr=%p,refc=%d,addr=%p,len=%ld,lkey=%08x,rkey=%08x\n", e, + //key, e->mr, e->refc, e->mr->addr, e->mr->length, e->mr->lkey, e->mr->rkey); //__lru_queue_display(); @@ -280,8 +280,8 @@ static void ibcom_reg_mr_dereg(struct ibv_mr *mr) e = (struct ibcom_reg_mr_cache_entry_t *) ((unsigned long) mr - offset); e->refc--; - dprintf("ibcom_reg_mr_dereg,entry=%p,mr=%p,addr=%p,refc=%d,offset=%lx\n", e, mr, e->mr->addr, - e->refc, offset); + //dprintf("ibcom_reg_mr_dereg,entry=%p,mr=%p,addr=%p,refc=%d,offset=%lx\n", e, mr, e->mr->addr, + //e->refc, offset); } void ibcom_RegisterCacheInit() http://git.mpich.org/mpich.git/commitdiff/e87f7e4e46fab8fc58a3f82f0e986760c… commit e87f7e4e46fab8fc58a3f82f0e986760c1a37b33 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Mon Dec 9 09:24:04 2013 +0900 Modify credit information packet rate Modify the rate at which an control packet of notifying flow-control credit information is sent to three halves of the rate at which flow-control credit information is embedded into a packet. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h index 2d73887..0935205 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h @@ -222,7 +222,7 @@ extern struct ibv_cq *rc_shared_scq_scratch_pad; #define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_STATE_LW 2 #define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_HW /*1*/(((IBCOM_RDMABUF_NSEG)>>4) == 0 ? 1 : ((IBCOM_RDMABUF_NSEG)>>4)) #define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_LW (((IBCOM_RDMABUF_NSEG)>>2)) /*12*/ /* receiver tries to notify sender the number of releases when receiver find not-noticed releases of more than this number */ -#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) /*(notify_rate + (notify_rate>>1))*/(notify_rate) /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */ +#define IBCOM_RDMABUF_OCCUPANCY_NOTIFY_RATE_DELAY_MULTIPLIER(notify_rate) (notify_rate + (notify_rate>>1)) /* (notify_rate) */ /* send seq_num to the sender side if there is no chance to embed seq_num into a packet bound for the sender side for this number of release events */ #define IBCOM_NBUF_UD 2 /* number of <addr, sz, lkey, rkey> */ #define IBCOM_UDWR_FROM 0 /* index to UD-write-from buffer */ http://git.mpich.org/mpich.git/commitdiff/f6243629523ab6501cceb336d64f20377… commit f6243629523ab6501cceb336d64f2037700d9059 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Mon Dec 9 09:22:30 2013 +0900 Correct reference overrun in debug message Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c index 9a6585d..585ba27 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c @@ -241,7 +241,7 @@ int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, REQ_FIELD(req, lmt_tail) = s_cookie_buf->tail; dprintf("lmt_start_recv,mem-tail=%p,%02x\n", write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t), - *((uint32_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t)))); + *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t)))); //dprintf("lmt_start_recv,sendq_empty=%d,ncom=%d,ncqe=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom < IBCOM_MAX_SQ_CAPACITY, MPID_nem_dcfa_ncqe < IBCOM_MAX_CQ_CAPACITY); http://git.mpich.org/mpich.git/commitdiff/e7de1763a0ba7fcddd9ee1ad5aec7961e… commit e7de1763a0ba7fcddd9ee1ad5aec7961ea281bef Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Thu Dec 12 15:51:08 2013 +0900 Increase max number of connections Increase the maxiumum number of connections whose message arrival is detected by polling. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h index 2f7afa8..2ee3bbb 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h @@ -368,7 +368,7 @@ extern MPID_nem_dcfa_vc_area *MPID_nem_dcfa_debug_current_vc_dcfa; extern uint8_t MPID_nem_dcfa_lmt_tail_addr_cbf[MPID_nem_dcfa_cbf_nslot * MPID_nem_dcfa_cbf_bitsperslot / 8]; -#define MPID_NEM_DCFA_MAX_POLLINGSET 64 +#define MPID_NEM_DCFA_MAX_POLLINGSET 65536 /* xfer.c manages memory region using memid */ #define MPID_NEM_DCFA_MEMID_RDMA 0 http://git.mpich.org/mpich.git/commitdiff/0fcd7852884622a8869ed767fd634423d… commit 0fcd7852884622a8869ed767fd634423d5092523 Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Thu Dec 12 15:46:14 2013 +0900 Eliminate --enable-strict warnings and errors Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c index 3baf664..5b10045 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c @@ -42,6 +42,6 @@ int MPID_nem_dcfa_finalize(void) fn_exit: return mpi_errno; - fn_fail: + //fn_fail: goto fn_exit; } diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c index 18df872..a8c308e 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c @@ -17,6 +17,8 @@ #include <sys/shm.h> #include <sys/types.h> #include <assert.h> +#include <linux/mman.h> /* make it define MAP_ANONYMOUS */ +#include <sys/mman.h> //#define DEBUG_IBCOM #ifdef dprintf /* avoid redefinition with src/mpid/ch3/include/mpidimpl.h */ @@ -694,14 +696,13 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc) (struct ibv_send_wr *) malloc(sizeof(struct ibv_send_wr) * IBCOM_RC_SR_NTEMPLATE); memset(conp->icom_sr, 0, sizeof(struct ibv_send_wr) * IBCOM_RC_SR_NTEMPLATE); - int i; for (i = 0; i < IBCOM_SMT_INLINE_NCHAIN; i++) { /* SGE (RDMA-send-from memory) template */ #ifdef HAVE_LIBDCFA memset(&(conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); #else - struct ibv_sge *sge = + sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_SMT_INLINE_INITIATOR_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SMT_INLINE_INITIATOR_NSGE); #endif @@ -723,7 +724,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc) memset(&(conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); #else - struct ibv_sge *sge = + sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_SMT_NOINLINE_INITIATOR_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SMT_NOINLINE_INITIATOR_NSGE); @@ -742,7 +743,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc) memset(&(conp->icom_sr[IBCOM_LMT_INITIATOR].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); #else - struct ibv_sge *sge = + sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_LMT_INITIATOR_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_LMT_INITIATOR_NSGE); #endif @@ -802,7 +803,7 @@ int ibcomOpen(int ib_port, int ibcom_open_flag, int *condesc) memset(&(conp->icom_sr[IBCOM_SCRATCH_PAD_INITIATOR].sg_list[0]), 0, sizeof(struct ibv_sge) * WR_SG_NUM); #else - struct ibv_sge *sge = + sge = (struct ibv_sge *) malloc(sizeof(struct ibv_sge) * IBCOM_SCRATCH_PAD_INITIATOR_NSGE); memset(sge, 0, sizeof(struct ibv_sge) * IBCOM_SCRATCH_PAD_INITIATOR_NSGE); @@ -1100,7 +1101,7 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void * num_sge = 0; void *buf_from = - conp->icom_mem[IBCOM_RDMAWR_FROM] + + (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_FROM] + IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf_from; @@ -1109,9 +1110,9 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void * /* memcpy hdr is needed because hdr resides in stack when sending close-VC command */ /* memcpy is performed onto IBCOM_RDMAWR_FROM buffer */ - void *hdr_copy = buf_from + sizeof(sz_hdrmagic_t); + void *hdr_copy = (uint8_t *) buf_from + sizeof(sz_hdrmagic_t); memcpy(hdr_copy, prefix, sz_prefix); - memcpy(hdr_copy + sz_prefix, hdr, sz_hdr); + memcpy((uint8_t *) hdr_copy + sz_prefix, hdr, sz_hdr); #ifdef HAVE_LIBDCFA conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = (uint64_t) sz_hdrmagic; conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].addr = @@ -1145,7 +1146,7 @@ int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void * int sz_pad = sz_data_pow2 - (sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_data); tailmagic_t *tailmagic = - (tailmagic_t *) (buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad); + (tailmagic_t *) ((uint8_t *) buf_from + sizeof(sz_hdrmagic_t) + sz_prefix + sz_hdr + sz_pad); tailmagic->magic = IBCOM_MAGIC; #ifdef HAVE_LIBDCFA conp->icom_sr[IBCOM_SMT_NOINLINE].sg_list[num_sge].mic_addr = @@ -1240,7 +1241,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void * printf("ibcom_isend_chain,icom_connected==0\n")); void *buf_from = - conp->icom_mem[IBCOM_RDMAWR_FROM] + + (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_FROM] + IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); /* make a tail-magic position is in a fixed set */ @@ -1266,7 +1267,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void * sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf_from; sz_hdrmagic->sz = sumsz; sz_hdrmagic->magic = IBCOM_MAGIC; - memcpy(buf_from + sizeof(sz_hdrmagic_t), hdr, sz_hdr); + memcpy((uint8_t *) buf_from + sizeof(sz_hdrmagic_t), hdr, sz_hdr); #ifdef HAVE_LIBDCFA conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = (uint64_t) buf_from; @@ -1278,7 +1279,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void * conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = (uint64_t) buf_from; #endif - buf_from += sizeof(sz_hdrmagic_t) + sz_hdr; + buf_from = (uint8_t *) buf_from + sizeof(sz_hdrmagic_t) + sz_hdr; conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sizeof(sz_hdrmagic_t) + sz_hdr; sz_used += sizeof(sz_hdrmagic_t) + sz_hdr; @@ -1328,7 +1329,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void * //tscs = MPID_nem_dcfa_rdtsc(); if (i == IBCOM_SMT_INLINE_NCHAIN - 1) { /* append tailmagic */ int sz_pad = sz_data_pow2 - sz_data; - tailmagic_t *tailmagic = (tailmagic_t *) (buf_from + sz_pad); + tailmagic_t *tailmagic = (tailmagic_t *) ((uint8_t *) buf_from + sz_pad); tailmagic->magic = IBCOM_MAGIC; #ifdef HAVE_LIBDCFA conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].mic_addr = @@ -1369,7 +1370,7 @@ int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void * conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].addr = (uint64_t) buf_from; #endif - buf_from += sz_pad; + buf_from = (uint8_t *) buf_from + sz_pad; conp->icom_sr[IBCOM_SMT_INLINE_CHAINED0 + i].sg_list[num_sge].length = sz_pad; sz_used += sz_pad; IBCOM_ERR_CHKANDJUMP(sz_used != IBCOM_INLINE_DATA, -1, @@ -1951,7 +1952,7 @@ int ibcom_mem_rdmawr_from(int condesc, void **out) RANGE_CHECK_WITH_ERROR(condesc, conp); *out = - conp->icom_mem[IBCOM_RDMAWR_FROM] + + (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_FROM] + IBCOM_RDMABUF_SZSEG * (conp->sseq_num % IBCOM_RDMABUF_NSEG); fn_exit: @@ -1966,7 +1967,7 @@ int ibcom_mem_rdmawr_to(int condesc, int seq_num, void **out) int ibcom_errno = 0; RANGE_CHECK_WITH_ERROR(condesc, conp); - *out = conp->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZSEG * (seq_num % IBCOM_RDMABUF_NSEG); + *out = (uint8_t *) conp->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZSEG * (seq_num % IBCOM_RDMABUF_NSEG); fn_exit: return ibcom_errno; @@ -2126,7 +2127,7 @@ int ibcom_obtain_pointer(int condesc, IbCom ** ibcom) goto fn_exit; } -void ibcomShow(int condesc) +static void ibcomShow(int condesc) { IbCom *conp; uint8_t *p; @@ -2167,7 +2168,7 @@ char *ibcom_strerror(int errno) } fn_exit: return r; - fn_fail: + //fn_fail: goto fn_exit; } diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h index b5970a8..2d73887 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h @@ -12,7 +12,6 @@ #include <unistd.h> #include <stdint.h> #include <sys/types.h> -#include <sys/mman.h> #include "mpid_nem_impl.h" //#define DEBUG_ON 1 @@ -322,6 +321,9 @@ extern int ibcom_rts(int condesc, int remote_qpnum, uint16_t remote_lid, union i extern int ibcom_reg_mr_connect(int condesc, void *rmem, int rkey); extern int ibcom_isend(int condesc, uint64_t wr_id, void *prefix, int sz_prefix, void *hdr, int sz_hdr, void *data, int sz_data, int *copied); +extern int ibcom_isend_chain(int condesc, uint64_t wr_id, void *hdr, int sz_hdr, void *data, + int sz_data); +extern int ibcom_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset, int sz, void *laddr); //extern int ibcom_isend(int condesc, uint64_t wr_id, void* hdr, int sz_hdr, void* data, int sz_data); extern int ibcom_irecv(int condesc, uint64_t wr_id); extern int ibcom_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_lid, @@ -329,14 +331,20 @@ extern int ibcom_udsend(int condesc, union ibv_gid *remote_gid, uint16_t remote_ extern int ibcom_udrecv(int condesc); extern int ibcom_lrecv(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, void *laddr); +extern int ibcom_put_lmt(int condesc, uint64_t wr_id, void *raddr, int sz_data, uint32_t rkey, + void *laddr); extern int ibcom_poll_cq(int which_cq, struct ibv_wc *wc, int *result); +extern int ibcom_obtain_pointer(int condesc, IbCom ** ibcom); + /* for dcfa_reg_mr.c */ extern int ibcom_reg_mr(void *addr, int len, struct ibv_mr **mr); +extern int ibcom_dereg_mr(struct ibv_mr *mr); extern int ibcom_get_info_conn(int condesc, int key, void *out, uint32_t out_len); extern int ibcom_get_info_mr(int condesc, int memid, int key, void *out, int out_len); +extern int ibcom_sseq_num_get(int condesc, int *seq_num); extern int ibcom_lsr_seq_num_tail_get(int condesc, int **seq_num); extern int ibcom_rsr_seq_num_tail_get(int condesc, int **seq_num); extern int ibcom_rsr_seq_num_tail_last_sent_get(int condesc, int **seq_num); @@ -345,7 +353,7 @@ extern int ibcom_rdmabuf_occupancy_notify_rstate_get(int condesc, int **rstate); extern int ibcom_rdmabuf_occupancy_notify_lstate_get(int condesc, int **lstate); extern int ibcomMemInfo(int, int, void **, size_t *, int *); -extern char *ibcom_strerror(int); +extern char *ibcom_strerror(int errno); extern int dflag; extern int ibcom_mem_rdmawr_from(int condesc, void **out); @@ -354,7 +362,8 @@ extern int ibcom_mem_udwr_from(int condesc, void **out); extern int ibcom_mem_udwr_to(int condesc, void **out); /* dcfa_reg_mr.c */ -extern void ibcom_RegisterCacheInit(); +extern void ibcom_RegisterCacheInit(void); +extern void ibcom_RegisterCacheDestroy(void); extern struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len); /* dcfa_ctlmsg.c */ diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h index 82fa93c..2f7afa8 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h @@ -12,6 +12,8 @@ #include "dcfa_ibcom.h" #include <sys/types.h> #include <errno.h> +#include <linux/mman.h> /* make it define MAP_ANONYMOUS */ +#include <sys/mman.h> #define LMT_GET_CQE /* detect RDMA completion by CQE */ //#define LMT_PUT_DONE @@ -276,12 +278,13 @@ static inline int MPID_nem_dcfa_cbf_would_overflow(uint64_t addr, uint8_t * arra } /* functions */ -uint64_t MPID_nem_dcfa_rdtsc(); +uint8_t MPID_nem_dcfa_rand(void); +uint64_t MPID_nem_dcfa_rdtsc(void); int MPID_nem_dcfa_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p); int MPID_nem_dcfa_finalize(void); -int MPID_nem_dcfa_drain_scq(); -int MPID_nem_dcfa_drain_scq_lmt_put(); -int MPID_nem_dcfa_drain_scq_scratch_pad(); +int MPID_nem_dcfa_drain_scq(int dont_call_progress); +int MPID_nem_dcfa_drain_scq_lmt_put(void); +int MPID_nem_dcfa_drain_scq_scratch_pad(void); int MPID_nem_dcfa_poll(int in_blocking_poll); int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc); @@ -307,8 +310,11 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa); /* CH3--lmt send/recv functions */ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts_pkt, struct MPID_Request *req); +int MPID_nem_dcfa_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, + void *write_to_buf); int MPID_nem_dcfa_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV s_cookie); int MPID_nem_dcfa_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *req, MPID_IOV cookie); +int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req); int MPID_nem_dcfa_lmt_done_send(struct MPIDI_VC *vc, struct MPID_Request *req); int MPID_nem_dcfa_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *req); int MPID_nem_dcfa_lmt_vc_terminated(struct MPIDI_VC *vc); @@ -500,13 +506,12 @@ int pkt_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * bu _pkt->subtype = MPIDI_NEM_DCFA_REPLY_SEQ_NUM; \ \ int *rsr_seq_num_tail; \ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); \ - ibcom_errno = ibcom_rsr_seq_num_tail_get(vc_dcfa->sc->fd, &rsr_seq_num_tail); \ + ibcom_errno = ibcom_rsr_seq_num_tail_get(VC_FIELD(vc, sc->fd), &rsr_seq_num_tail); \ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rsr_seq_num_tail_get"); \ _pkt->seq_num_tail = *rsr_seq_num_tail; \ \ int *rsr_seq_num_tail_last_sent; \ - ibcom_errno = ibcom_rsr_seq_num_tail_last_sent_get(vc_dcfa->sc->fd, &rsr_seq_num_tail_last_sent); \ + ibcom_errno = ibcom_rsr_seq_num_tail_last_sent_get(VC_FIELD(vc, sc->fd), &rsr_seq_num_tail_last_sent); \ MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_rsr_seq_num_tail_last_sent_get"); \ *rsr_seq_num_tail_last_sent = *rsr_seq_num_tail; \ \ @@ -561,17 +566,16 @@ int pkt_DONE_handler(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, MPIDI_msg_sz_t * bu #define MPID_nem_dcfa_lmt_send_GET_DONE(vc, rreq) do { \ MPID_PKT_DECL_CAST(_upkt, MPID_nem_dcfa_pkt_lmt_get_done_t, _done_pkt); \ MPID_Request *_done_req; \ - MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); \ \ MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending rndv DONE packet"); \ MPIDI_Pkt_init(_done_pkt, MPIDI_NEM_PKT_NETMOD); \ _done_pkt->subtype = MPIDI_NEM_DCFA_PKT_LMT_GET_DONE;\ _done_pkt->req_id = (rreq)->ch.lmt_req_id; \ /* embed SR occupancy information */ \ - _done_pkt->seq_num_tail = vc_dcfa->ibcom->rsr_seq_num_tail; \ + _done_pkt->seq_num_tail = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ \ /* remember the last one sent */ \ - vc_dcfa->ibcom->rsr_seq_num_tail_last_sent = vc_dcfa->ibcom->rsr_seq_num_tail; \ + VC_FIELD(vc, ibcom->rsr_seq_num_tail_last_sent) = VC_FIELD(vc, ibcom->rsr_seq_num_tail); \ \ mpi_errno = MPIDI_CH3_iStartMsg((vc), _done_pkt, sizeof(*_done_pkt), &_done_req); \ MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**donepkt"); \ diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c index 5aebe8e..871feea 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c @@ -112,7 +112,7 @@ uint64_t MPID_nem_dcfa_rdtsc() #define FUNCNAME MPID_nem_dcfa_kvs_put_binary #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, int length) +static int MPID_nem_dcfa_kvs_put_binary(int from, const char *postfix, const uint8_t * buf, int length) { int mpi_errno = MPI_SUCCESS; int pmi_errno; @@ -148,7 +148,7 @@ int MPID_nem_dcfa_kvs_put_binary(int from, const char *postfix, const uint8_t * #define FUNCNAME MPID_nem_dcfa_kvs_get_binary #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_kvs_get_binary(int from, const char *postfix, char *buf, int length) +static int MPID_nem_dcfa_kvs_get_binary(int from, const char *postfix, char *buf, int length) { int mpi_errno = MPI_SUCCESS; int pmi_errno; @@ -185,6 +185,8 @@ int MPID_nem_dcfa_kvs_get_binary(int from, const char *postfix, char *buf, int l goto fn_exit; } +static int MPID_nem_dcfa_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p); + #undef FUNCNAME #define FUNCNAME MPID_nem_dcfa_init #undef FCNAME @@ -194,7 +196,7 @@ int MPID_nem_dcfa_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val int mpi_errno = MPI_SUCCESS; int ibcom_errno = 0, pmi_errno; int ret; - int i, j; + int i, j, k; int ib_port = 1; MPIU_CHKPMEM_DECL(7); @@ -380,13 +382,11 @@ int MPID_nem_dcfa_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_dcfa_kvs_put_binary"); - { - dprintf("dcfa_init,scratch pad,gid "); - int i; - for (i = 0; i < 16; i++) { - dprintf("%02x", (int) my_gid.raw[i]); - } dprintf("\n"); + dprintf("dcfa_init,scratch pad,gid "); + for (k = 0; k < 16; k++) { + dprintf("%02x", (int) my_gid.raw[k]); } + dprintf("\n"); } /* put bc/me/sp/qpn/you */ @@ -662,12 +662,12 @@ int MPID_nem_dcfa_get_business_card(int my_rank, char **bc_val_p, int *val_max_s #define FUNCNAME MPID_nem_dcfa_announce_network_addr #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p) +static int MPID_nem_dcfa_announce_network_addr(int my_rank, char **bc_val_p, int *val_max_sz_p) { int mpi_errno = MPI_SUCCESS; int str_errno = MPIU_STR_SUCCESS; int ibcom_errno; - int i, nranks; + int i, j, nranks; uint32_t my_qpnum; uint16_t my_lid; @@ -714,15 +714,14 @@ int MPID_nem_dcfa_announce_network_addr(int my_rank, char **bc_val_p, int *val_m ibcom_errno = ibcom_get_info_conn(MPID_nem_dcfa_conns[i].fd, IBCOM_INFOKEY_PORT_GID, &my_gid, sizeof(union ibv_gid)); - { - dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); - dprintf("get_business_card,sz=%ld,my_gid=", sizeof(union ibv_gid)); - int i; - for (i = 0; i < 16; i++) { - dprintf("%02x", (int) my_gid.raw[i]); - } - dprintf("\n"); + + dprintf("get_business_card,val_max_sz=%d\n", *val_max_sz_p); + dprintf("get_business_card,sz=%ld,my_gid=", sizeof(union ibv_gid)); + for (j = 0; j < 16; j++) { + dprintf("%02x", (int) my_gid.raw[j]); } + dprintf("\n"); + MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**ibcom_get_info_conn"); mpi_errno = @@ -820,7 +819,7 @@ int MPID_nem_dcfa_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc dprintf("toroot,%d->%d", MPID_nem_dcfa_myrank, new_vc->pg_rank); /* not implemented */ - fn_exit: + //fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_CONNECT_TO_ROOT); return MPI_SUCCESS; } @@ -884,7 +883,7 @@ int MPID_nem_dcfa_vc_init(MPIDI_VC_t * vc) int ntrial = 0; volatile ibcom_qp_state_t *rstate = - (ibcom_qp_state_t *) (ibcom_scratch_pad->icom_mem[IBCOM_SCRATCH_PAD_TO] + + (ibcom_qp_state_t *) ((uint8_t *) ibcom_scratch_pad->icom_mem[IBCOM_SCRATCH_PAD_TO] + vc->pg_rank * sizeof(ibcom_qp_state_t)); dprintf("dcfa_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate)); while (rstate->state != IBCOM_QP_STATE_RTR) { diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c index c145343..9a6585d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c @@ -95,7 +95,7 @@ int MPID_nem_dcfa_lmt_initiate_lmt(struct MPIDI_VC *vc, union MPIDI_CH3_Pkt *rts /* preserve and put tail, because tail magic is written on the tail of payload * because we don't want to add another SGE or RDMA command */ MPIU_Assert(((MPID_nem_pkt_lmt_rts_t *) rts_pkt)->data_sz == data_sz); - s_cookie_buf->tail = *((uint8_t *) (write_from_buf + data_sz - sizeof(uint8_t))); + s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t))); /* prepare magic */ //*((uint32_t*)(write_from_buf + data_sz - sizeof(tailmagic_t))) = IBCOM_MAGIC; @@ -360,7 +360,7 @@ int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) } //assert(dt_true_lb == 0); - uint8_t *tailp = (uint8_t *) (write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t)); + uint8_t *tailp = (uint8_t *) ((uint8_t *) write_from_buf /*+ dt_true_lb */ + data_sz - sizeof(uint8_t)); #if 0 *is_end_flag_same = (r_cookie_buf->tail == *tailp) ? 1 : 0; #else @@ -368,7 +368,7 @@ int MPID_nem_dcfa_lmt_switch_send(struct MPIDI_VC *vc, struct MPID_Request *req) REQ_FIELD(req, lmt_sender_tail) = *tailp; dprintf("lmt_switch_send,tail on sender=%02x,tail onreceiver=%02x,req=%p\n", *tailp, r_cookie_buf->tail, req); - uint8_t *tail_wordp = (uint8_t *) (write_from_buf + data_sz - sizeof(uint32_t) * 2); + uint8_t *tail_wordp = (uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint32_t) * 2); dprintf("lmt_switch_send,tail on sender=%d\n", *tail_wordp); fflush(stdout); #endif @@ -398,7 +398,7 @@ int MPID_nem_dcfa_lmt_handle_cookie(struct MPIDI_VC *vc, struct MPID_Request *re fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_HANDLE_COOKIE); return mpi_errno; - fn_fail: + //fn_fail: goto fn_exit; } @@ -526,7 +526,7 @@ int MPID_nem_dcfa_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq) fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_DONE_RECV); return mpi_errno; - fn_fail: + //fn_fail: goto fn_exit; } @@ -547,6 +547,6 @@ int MPID_nem_dcfa_lmt_vc_terminated(struct MPIDI_VC *vc) fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_LMT_VC_TERMINATED); return mpi_errno; - fn_fail: + //fn_fail: goto fn_exit; } diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c index 4ed0e39..302808d 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c @@ -23,12 +23,11 @@ static int entered_drain_scq = 0; #define MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET MPID_nem_dcfa_send_progress(vc_dcfa); #else #define MPID_NEM_DCFA_SEND_PROGRESS_POLLINGSET { \ - int i; \ - for(i = 0; i < MPID_nem_dcfa_npollingset; i++) { \ - MPIDI_VC_t *vc = MPID_nem_dcfa_pollingset[i]; \ - MPID_nem_dcfa_vc_area *_vc_dcfa = VC_DCFA(vc); \ + int n; \ + for(n = 0; n < MPID_nem_dcfa_npollingset; n++) { \ + MPIDI_VC_t *vc_n = MPID_nem_dcfa_pollingset[n]; \ /*MPID_nem_dcfa_debug_current_vc_dcfa = vc_dcfa;*/ \ - MPID_nem_dcfa_send_progress(_vc_dcfa); \ + MPID_nem_dcfa_send_progress(VC_DCFA(vc_n)); \ } \ } #endif @@ -195,15 +194,15 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) IBCOM_RDMABUF_NSEG); MPID_Request *sreq = MPID_nem_dcfa_sendq_head(vc_dcfa->sendq); - int msg_type = MPIDI_Request_get_msg_type(sreq); + int msg_type_sreq = MPIDI_Request_get_msg_type(sreq); - if (sreq->kind == MPID_REQUEST_SEND && msg_type == MPIDI_REQUEST_EAGER_MSG) { + if (sreq->kind == MPID_REQUEST_SEND && msg_type_sreq == MPIDI_REQUEST_EAGER_MSG) { dprintf("drain_scq,eager-send,head is eager-send\n"); } - else if (sreq->kind == MPID_REQUEST_RECV && msg_type == MPIDI_REQUEST_RNDV_MSG) { + else if (sreq->kind == MPID_REQUEST_RECV && msg_type_sreq == MPIDI_REQUEST_RNDV_MSG) { dprintf("drain_scq,eager-send,head is lmt RDMA-read\n"); } - else if (sreq->kind == MPID_REQUEST_SEND && msg_type == MPIDI_REQUEST_RNDV_MSG) { + else if (sreq->kind == MPID_REQUEST_SEND && msg_type_sreq == MPIDI_REQUEST_RNDV_MSG) { dprintf("drain_scq,eager-send,head is lmt RDMA-write\n"); } } @@ -360,12 +359,12 @@ int MPID_nem_dcfa_drain_scq(int dont_call_progress) vc_dcfa->ibcom->lsr_seq_num_tail) < IBCOM_RDMABUF_NSEG); MPID_Request *sreq = MPID_nem_dcfa_sendq_head(vc_dcfa->sendq); - int msg_type = MPIDI_Request_get_msg_type(sreq); + int msg_type_sreq = MPIDI_Request_get_msg_type(sreq); - if (sreq->kind == MPID_REQUEST_SEND && msg_type == MPIDI_REQUEST_EAGER_MSG) { + if (sreq->kind == MPID_REQUEST_SEND && msg_type_sreq == MPIDI_REQUEST_EAGER_MSG) { dprintf("drain_scq,eager-send,head is eager-send\n"); } - else if (sreq->kind == MPID_REQUEST_RECV && msg_type == MPIDI_REQUEST_RNDV_MSG) { + else if (sreq->kind == MPID_REQUEST_RECV && msg_type_sreq == MPIDI_REQUEST_RNDV_MSG) { dprintf("drain_scq,eager-send,head is lmt\n"); } } @@ -580,7 +579,7 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) MPID_nem_dcfa_vc_area *vc_dcfa = VC_DCFA(vc); //dprintf("dcfa_poll,ld,rsr_seq_num_poll=%d\n", vc_dcfa->ibcom->rsr_seq_num_poll); volatile void *buf = - vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + + (uint8_t *) vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZSEG * ((uint32_t) vc_dcfa->ibcom->rsr_seq_num_poll % IBCOM_RDMABUF_NSEG); volatile sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf; if (sz_hdrmagic->magic != IBCOM_MAGIC) { @@ -600,7 +599,7 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) int sz_data_pow2; DCFA_NEM_SZ_DATA_POW2(sz_hdrmagic->sz); - volatile tailmagic_t *tailmagic = (tailmagic_t *) (buf + sz_data_pow2); + volatile tailmagic_t *tailmagic = (tailmagic_t *) ((uint8_t *) buf + sz_data_pow2); dprintf("poll,sz_data_pow2=%d,tailmagic=%p,sz=%d\n", sz_data_pow2, tailmagic, sz_hdrmagic->sz); int k = 0; //tsce = MPID_nem_dcfa_rdtsc(); printf("9,%ld\n", tsce - tscs); // 55 for 512-byte @@ -639,7 +638,7 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) #if 1 void *rsi; - for (rsi = (void *) buf; rsi < buf + sz_hdrmagic->sz; rsi += 64 * 4) { + for (rsi = (void *) buf; rsi < (uint8_t *) buf + sz_hdrmagic->sz; rsi = (uint8_t *) rsi + 64 * 4) { #ifdef __MIC__ __asm__ __volatile__ ("movq %0, %%rsi;" @@ -657,11 +656,11 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) } #endif - MPIDI_CH3_Pkt_eager_send_t *pkt = (MPIDI_CH3_Pkt_eager_send_t *) (buf + sizeof(sz_hdrmagic_t)); + MPIDI_CH3_Pkt_eager_send_t *pkt = (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t)); MPIU_Assert(sz_hdrmagic->sz >= sizeof(sz_hdrmagic_t) + sizeof(MPIDI_CH3_Pkt_t) + sizeof(tailmagic_t)); MPIDI_CH3_Pkt_eager_send_t *pkt2 = - (MPIDI_CH3_Pkt_eager_send_t *) (buf + sizeof(sz_hdrmagic_t) + + (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t) + sizeof(MPID_nem_dcfa_pkt_prefix_t)); dprintf ("handle_pkt,before,%d<-%d,id=%d,pkt->type=%d,pcc=%d,MPIDI_NEM_PKT_END=%d,pkt=%p,subtype=%d\n", @@ -670,7 +669,7 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) ((MPID_nem_pkt_netmod_t *) pkt)->subtype); /* see MPIDI_CH3_PktHandler_EagerSend (in src/mpid/ch3/src/ch3u_eager.c) */ mpi_errno = - MPID_nem_handle_pkt(vc, (char *) (buf + sizeof(sz_hdrmagic_t)), + MPID_nem_handle_pkt(vc, (char *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t)), (MPIDI_msg_sz_t) (sz_hdrmagic->sz - sizeof(sz_hdrmagic_t) - sizeof(tailmagic_t))); if (mpi_errno) { @@ -716,8 +715,8 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) dprintf("poll_eager,released,type=%d,MPIDI_NEM_DCFA_REPLY_SEQ_NUM=%d\n", pkt->type, MPIDI_NEM_DCFA_REPLY_SEQ_NUM); MPID_nem_dcfa_recv_buf_released(vc, - (void *) buf + sizeof(sz_hdrmagic_t) + - sizeof(MPIDI_CH3_Pkt_t)); + (void *) ((uint8_t *) buf + sizeof(sz_hdrmagic_t) + + sizeof(MPIDI_CH3_Pkt_t))); } else { if (sz_hdrmagic->sz == @@ -738,7 +737,6 @@ int MPID_nem_dcfa_poll_eager(MPIDI_VC_t * vc) vc_dcfa->ibcom->rsr_seq_num_poll += 1; dprintf("dcfa_poll,inc,rsr_seq_num_poll=%d\n", vc_dcfa->ibcom->rsr_seq_num_poll); - out: fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_POLL_EAGER); return mpi_errno; @@ -803,7 +801,7 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) //assert(REQ_FIELD(rreq, lmt_dt_true_lb) == 0); volatile uint8_t *tailmagic = - (uint8_t *) (write_to_buf /*+ REQ_FIELD(rreq, lmt_dt_true_lb) */ + + (uint8_t *) ((uint8_t *) write_to_buf /*+ REQ_FIELD(rreq, lmt_dt_true_lb) */ + rreq->ch.lmt_data_sz - sizeof(uint8_t)); uint8_t lmt_tail = REQ_FIELD(rreq, lmt_tail); @@ -1002,7 +1000,6 @@ int MPID_nem_dcfa_poll(int in_blocking_poll) #endif //if (in_blocking_poll) { goto prev; } - out: fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_DCFA_POLL); return mpi_errno; @@ -1117,10 +1114,10 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) MPIU_Assert(vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] <= user_data && user_data < vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZ); unsigned long mod = - (unsigned long) (user_data - - vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO]) & (IBCOM_RDMABUF_SZSEG - 1); + (unsigned long) ((uint8_t *) user_data - + (uint8_t *) vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO]) & (IBCOM_RDMABUF_SZSEG - 1); - void *buf = (void *) (user_data - mod); + void *buf = (void *) ((uint8_t *) user_data - mod); //dprintf("recv_buf_released,clearing,buf=%p\n", buf); sz_hdrmagic_t *sz_hdrmagic = (sz_hdrmagic_t *) buf; @@ -1134,7 +1131,7 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) offset ? ((((offset + 1) << 1) - 1) > DCFA_NEM_MAX_DATA_POW2 ? DCFA_NEM_MAX_DATA_POW2 : (((offset + 1) << 1) - 1)) : 15) { - volatile tailmagic_t *ptr = (tailmagic_t *) (buf + offset); + volatile tailmagic_t *ptr = (tailmagic_t *) ((uint8_t *) buf + offset); MPIU_Assert(vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] <= ptr && ptr < vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO] + IBCOM_RDMABUF_SZ); ptr->magic = 0 /*0xde */ ; @@ -1148,8 +1145,8 @@ int MPID_nem_dcfa_recv_buf_released(struct MPIDI_VC *vc, void *user_data) #if 1 /* moving from dcfa_poll */ /* mark that one eager-send RDMA-write-to buffer has been released */ int index_slot = - (unsigned long) (user_data - - vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO]) / IBCOM_RDMABUF_SZSEG; + (unsigned long) ((uint8_t *) user_data - + (uint8_t *) vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO]) / IBCOM_RDMABUF_SZSEG; MPIU_Assert(0 <= index_slot && index_slot < IBCOM_RDMABUF_NSEG); //dprintf("user_data=%p,mem=%p,sub=%08lx,index_slot=%d\n", user_data, vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO], (unsigned long)user_data - (unsigned long)vc_dcfa->ibcom->icom_mem[IBCOM_RDMAWR_TO], index_slot); //dprintf("index_slot=%d,released=%016lx\n", index_slot, vc_dcfa->ibcom->rsr_seq_num_released[index_slot / 64]); @@ -1281,7 +1278,7 @@ int MPID_nem_dcfa_PktHandler_EagerSend(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_eager_send_t *ch3_pkt = - (MPIDI_CH3_Pkt_eager_send_t *) ((void *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_eager_send_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1375,7 +1372,7 @@ int MPID_nem_dcfa_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_put_t *ch3_pkt = - (MPIDI_CH3_Pkt_put_t *) ((void *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_put_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1443,7 +1440,7 @@ int MPID_nem_dcfa_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_accum_t *ch3_pkt = - (MPIDI_CH3_Pkt_accum_t *) ((void *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_accum_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1510,7 +1507,7 @@ int MPID_nem_dcfa_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_get_t *ch3_pkt = - (MPIDI_CH3_Pkt_get_t *) ((void *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; @@ -1576,7 +1573,7 @@ int MPID_nem_dcfa_PktHandler_GetResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt, { MPID_nem_dcfa_pkt_prefix_t *netmod_pkt = (MPID_nem_dcfa_pkt_prefix_t *) pkt; MPIDI_CH3_Pkt_get_t *ch3_pkt = - (MPIDI_CH3_Pkt_get_t *) ((void *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); + (MPIDI_CH3_Pkt_get_t *) ((uint8_t *) pkt + sizeof(MPID_nem_dcfa_pkt_prefix_t)); MPID_Request *rreq; int found; int complete; diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c index 3b08b0b..42fe244 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c @@ -115,7 +115,7 @@ __inline__ int ibcom_hash_func(char *addr) return (int) v; } -void ibcom_reg_mr_insert(struct ibcom_reg_mr_listnode_t *c, struct ibcom_reg_mr_listnode_t *e) +static void ibcom_reg_mr_insert(struct ibcom_reg_mr_listnode_t *c, struct ibcom_reg_mr_listnode_t *e) { struct ibcom_reg_mr_listnode_t *next; struct ibcom_reg_mr_listnode_t *prev; @@ -127,7 +127,7 @@ void ibcom_reg_mr_insert(struct ibcom_reg_mr_listnode_t *c, struct ibcom_reg_mr_ prev->lru_next = e; } -void ibcom_reg_mr_unlink(struct ibcom_reg_mr_listnode_t *e) +static void ibcom_reg_mr_unlink(struct ibcom_reg_mr_listnode_t *e) { struct ibcom_reg_mr_listnode_t *next, *prev; next = e->lru_next; @@ -198,7 +198,7 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) e = (struct ibcom_reg_mr_cache_entry_t *) e->lru_next, way++) { //dprintf("e=%p, e->hash_next=%p\n", e, e->lru_next); - if (e->addr <= addr_aligned && addr_aligned + len_aligned <= e->addr + e->len) { + if (e->addr <= addr_aligned && (uint8_t *) addr_aligned + len_aligned <= (uint8_t *) e->addr + e->len) { dprintf ("ibcom_reg_mr_fetch,hit,entry addr=%p,len=%d,mr addr=%p,len=%ld,requested addr=%p,len=%d\n", e->addr, e->len, e->mr->addr, e->mr->length, addr, len); @@ -216,7 +216,7 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) dprintf("ibcom_reg_mr,evict,entry addr=%p,len=%d,mr addr=%p,len=%ld\n", e->addr, e->len, e->mr->addr, e->mr->length); - int ibcom_errno = ibcom_dereg_mr(victim->mr); + ibcom_errno = ibcom_dereg_mr(victim->mr); if (ibcom_errno) { printf("mrcache,ibcom_dereg_mr\n"); goto fn_fail; @@ -271,7 +271,7 @@ struct ibv_mr *ibcom_reg_mr_fetch(void *addr, int len) #endif } -void ibcom_reg_mr_dereg(struct ibv_mr *mr) +static void ibcom_reg_mr_dereg(struct ibv_mr *mr) { struct ibcom_reg_mr_cache_entry_t *e; diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c index bf5b580..dd6f120 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c @@ -636,7 +636,7 @@ int MPID_nem_dcfa_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr #define FUNCNAME MPID_nem_dcfa_SendNoncontig_core #undef FCNAME #define FCNAME MPIDI_QUOTE(FUNCNAME) -int MPID_nem_dcfa_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, +static int MPID_nem_dcfa_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; @@ -1046,15 +1046,13 @@ int MPID_nem_dcfa_send_progress(MPID_nem_dcfa_vc_area * vc_dcfa) MPID_Request *tmp_sreq = sreq; sreq = MPID_nem_dcfa_sendq_next(sreq); goto next_unlinked; - next: + //next: prev_sreq = sreq; sreq = MPID_nem_dcfa_sendq_next(sreq); next_unlinked:; } while (sreq); } - out: - //dprintf("send_progress,exit,sendq_empty=%d,ncom=%d,ncqe=%d,rdmabuf_occ=%d\n", MPID_nem_dcfa_sendq_empty(vc_dcfa->sendq), vc_dcfa->ibcom->ncom, MPID_nem_dcfa_ncqe, MPID_nem_dcfa_diff32(vc_dcfa->ibcom->sseq_num, vc_dcfa->ibcom->lsr_seq_num_tail)); fn_exit: ----------------------------------------------------------------------- Summary of changes: README.vin | 22 +- src/mpid/ch3/channels/nemesis/netmod/Makefile.mk | 2 +- .../ch3/channels/nemesis/netmod/dcfa/Makefile.mk | 22 - .../channels/nemesis/netmod/dcfa/dcfa_finalize.c | 47 - .../ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c | 2216 -------------------- .../ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h | 414 ---- .../ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h | 642 ------ .../ch3/channels/nemesis/netmod/dcfa/dcfa_init.c | 1113 ---------- .../ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c | 552 ----- .../ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c | 2085 ------------------ .../ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c | 321 --- .../ch3/channels/nemesis/netmod/dcfa/dcfa_send.c | 1116 ---------- .../ch3/channels/nemesis/netmod/dcfa/errnames.txt | 53 - .../channels/nemesis/netmod/dcfa/subconfigure.m4 | 39 - .../ch3/channels/nemesis/netmod/ib/Makefile.mk | 23 + .../nemesis/netmod/{dcfa => ib}/cross_values.txt | 0 .../ch3/channels/nemesis/netmod/ib/errnames.txt | 53 + .../ch3/channels/nemesis/netmod/ib/ib_finalize.c | 47 + src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c | 2215 +++++++++++++++++++ src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h | 414 ++++ src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h | 642 ++++++ src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c | 1112 ++++++++++ src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c | 530 +++++ src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c | 2082 ++++++++++++++++++ .../ch3/channels/nemesis/netmod/ib/ib_reg_mr.c | 321 +++ src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c | 1126 ++++++++++ .../ch3/channels/nemesis/netmod/ib/subconfigure.m4 | 39 + src/mpid/ch3/channels/nemesis/subconfigure.m4 | 2 +- 28 files changed, 8617 insertions(+), 8633 deletions(-) delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/Makefile.mk delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_finalize.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_ibcom.h delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_impl.h delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_init.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_lmt.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_poll.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_reg_mr.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/dcfa_send.c delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/errnames.txt delete mode 100644 src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk rename src/mpid/ch3/channels/nemesis/netmod/{dcfa => ib}/cross_values.txt (100%) create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/errnames.txt create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_finalize.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_ibcom.h create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_impl.h create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_init.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_lmt.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_poll.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_reg_mr.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/ib_send.c create mode 100644 src/mpid/ch3/channels/nemesis/netmod/ib/subconfigure.m4 hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-77-g5711e32
by noreply＠mpich.org 18 Dec '13

18 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 5711e32d3597f48bc57d092d30b40950134fc40c (commit) from 1c70aae3f0ee9cafb88e5e640f04f60fe257dfce (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/5711e32d3597f48bc57d092d30b409501… commit 5711e32d3597f48bc57d092d30b40950134fc40c Author: Masamichi Takagi <masamichi.takagi(a)gmail.com> Date: Mon Dec 9 18:57:24 2013 +0900 Auto-detect IB libraries for the DCFA netmod The configure auto-detects IB libraries in the following order. 1. If libdcfa.so is found, it's selected automatically. 2. If libibverbs.so is found, it's selected automatically. Note that the search path can be specified by using --with-dcfa=<path> option. Signed-off-by: Pavan Balaji <balaji(a)mcs.anl.gov> diff --git a/src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 b/src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 index 9274887..b7a5d08 100644 --- a/src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 +++ b/src/mpid/ch3/channels/nemesis/netmod/dcfa/subconfigure.m4 @@ -17,14 +17,17 @@ AC_DEFUN([PAC_SUBCFG_BODY_]PAC_SUBCFG_AUTO_SUFFIX,[ AM_COND_IF([BUILD_NEMESIS_NETMOD_DCFA],[ AC_MSG_NOTICE([RUNNING CONFIGURE FOR ch3:nemesis:dcfa]) - AC_ARG_ENABLE(dcfa, [--enable-dcfa - use DCFA library instead of IB Verbs library for MPICH/DCFA/McKernel/MIC],,enable_dcfa=no) - if test "$enable_dcfa" = "yes" ; then - AC_MSG_NOTICE([--enable-dcfa detected]) - PAC_CHECK_HEADER_LIB_FATAL(dcfa, dcfa.h, dcfa, ibv_open_device) -# see confdb/aclocal_libs.m4 - else - PAC_CHECK_HEADER_LIB_FATAL(ib, infiniband/verbs.h, ibverbs, ibv_open_device) - fi + PAC_CHECK_HEADER_LIB(dcfa.h,dcfa,ibv_open_device,dcfa_found=yes,dcfa_found=no) + if test "${dcfa_found}" = "yes" ; then + AC_MSG_NOTICE([libdcfa is going to be linked.]) + else + PAC_CHECK_HEADER_LIB([infiniband/verbs.h],ibverbs,ibv_open_device,ibverbs_found=yes,ibverbs_found=no) + if test "${ibverbs_found}" = "yes" ; then + AC_MSG_NOTICE([libibverbs is going to be linked.]) + else + AC_MSG_ERROR([Internal error: neither ibverbs nor dcfa was found]) + fi + fi AC_DEFINE([MPID_NEM_DCFA_VERSION], ["0.9.0"], [Version of netmod/DCFA]) AC_DEFINE([MPID_NEM_DCFA_RELEASE_DATE], ["2013-11-18"], [Release date of netmod/DCFA]) ----------------------------------------------------------------------- Summary of changes: .../channels/nemesis/netmod/dcfa/subconfigure.m4 | 19 +++++++++++-------- 1 files changed, 11 insertions(+), 8 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-76-g1c70aae
by noreply＠mpich.org 18 Dec '13

18 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 1c70aae3f0ee9cafb88e5e640f04f60fe257dfce (commit) from e49ee8c03d72671d41616a07f818023e69dae365 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/1c70aae3f0ee9cafb88e5e640f04f60fe… commit 1c70aae3f0ee9cafb88e5e640f04f60fe257dfce Author: Bob Cernohous <bobc(a)us.ibm.com> Date: Tue Sep 10 12:16:47 2013 -0500 Test updates for ad_bglockless. Add some minor error checking to fail more obviously. Add MPITEST_FILE_ERRORS_THROW_EXCEPTIONS to turn on file i/o exceptions for cxx tests. (ibm) Issue 8847 (ibm) ec58bb29b7f8051973303364c6f469bb291773e3 Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> Signed-off-by: Rob Latham <robl(a)mcs.anl.gov> diff --git a/test/mpi/cxx/util/mtest.cxx b/test/mpi/cxx/util/mtest.cxx index ee5479f..a4689b4 100644 --- a/test/mpi/cxx/util/mtest.cxx +++ b/test/mpi/cxx/util/mtest.cxx @@ -76,6 +76,8 @@ void MTest_Init( void ) provided = MPI::Init_thread( threadLevel ); } + MPI::FILE_NULL.Set_errhandler(MPI::ERRORS_THROW_EXCEPTIONS); + /* Check for debugging control */ if (getenv( "MPITEST_DEBUG" )) { dbgflag = 1; diff --git a/test/mpi/f77/io/shpositionf.f b/test/mpi/f77/io/shpositionf.f index d403537..65a474a 100644 --- a/test/mpi/f77/io/shpositionf.f +++ b/test/mpi/f77/io/shpositionf.f @@ -40,6 +40,12 @@ C write if (i .eq. r + 1) then call mpi_file_write_shared( fh, i, 1, MPI_INTEGER, & MPI_STATUS_IGNORE, ierr ) + if (ierr .ne. MPI_SUCCESS) then + errs = errs + 1 + if (errs .le. 10) then + call MTestPrintError( ierr ) + endif + endif endif call mpi_barrier( comm, ierr ) call mpi_file_get_position_shared( fh, offset, ierr ) diff --git a/test/mpi/io/rdwrord.c b/test/mpi/io/rdwrord.c index 47f744e..245b266 100644 --- a/test/mpi/io/rdwrord.c +++ b/test/mpi/io/rdwrord.c @@ -16,7 +16,7 @@ static char MTEST_Descrip[] = "Test reading and writing ordered output"; int main( int argc, char *argv[] ) { int errs = 0; - int size, rank, i, *buf; + int size, rank, i, *buf, rc; MPI_File fh; MPI_Comm comm; MPI_Status status; @@ -32,8 +32,11 @@ int main( int argc, char *argv[] ) MPI_Comm_rank( comm, &rank ); buf = (int *)malloc( size * sizeof(int) ); buf[0] = rank; - MPI_File_write_ordered( fh, buf, 1, MPI_INT, &status ); - + rc = MPI_File_write_ordered( fh, buf, 1, MPI_INT, &status ); + if (rc) { + MTestPrintErrorMsg( "File_write_ordered", rc ); + errs++; + } /* make sure all writes finish before we seek/read */ MPI_Barrier(comm); ----------------------------------------------------------------------- Summary of changes: test/mpi/cxx/util/mtest.cxx | 2 ++ test/mpi/f77/io/shpositionf.f | 6 ++++++ test/mpi/io/rdwrord.c | 9 ++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, get-acc-from-sameer-3, deleted. v3.1rc2-75-ge49ee8c
by noreply＠mpich.org 18 Dec '13

18 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, get-acc-from-sameer-3 has been deleted was e49ee8c03d72671d41616a07f818023e69dae365 ----------------------------------------------------------------------- e49ee8c03d72671d41616a07f818023e69dae365 Implement MPID_Get_accumulate() ----------------------------------------------------------------------- hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-75-ge49ee8c
by noreply＠mpich.org 18 Dec '13

18 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via e49ee8c03d72671d41616a07f818023e69dae365 (commit) from 356762dbfb82359a673598c1e3577111eb4def99 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- ----------------------------------------------------------------------- Summary of changes: src/mpid/pamid/include/mpidi_datatypes.h | 2 + src/mpid/pamid/include/mpidi_prototypes.h | 19 + src/mpid/pamid/src/misc/mpid_unimpl.c | 8 - src/mpid/pamid/src/mpid_init.c | 24 + src/mpid/pamid/src/onesided/Makefile.mk | 1 + src/mpid/pamid/src/onesided/mpid_win_accumulate.c | 2 +- .../pamid/src/onesided/mpid_win_get_accumulate.c | 517 ++++++++++++++++++++ src/mpid/pamid/src/onesided/mpidi_onesided.h | 30 +- 8 files changed, 591 insertions(+), 12 deletions(-) create mode 100644 src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, get-acc-from-sameer-3, created. v3.1rc2-75-ge49ee8c
by noreply＠mpich.org 18 Dec '13

18 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, get-acc-from-sameer-3 has been created at e49ee8c03d72671d41616a07f818023e69dae365 (commit) - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/e49ee8c03d72671d41616a07f818023e6… commit e49ee8c03d72671d41616a07f818023e69dae365 Author: Sameer Kumar <sameerk(a)us.ibm.com> Date: Wed Dec 18 04:40:40 2013 -0600 Implement MPID_Get_accumulate() Does not include the win request tls optimization Signed-off-by: Michael Blocksome <blocksom(a)us.ibm.com> diff --git a/src/mpid/pamid/include/mpidi_datatypes.h b/src/mpid/pamid/include/mpidi_datatypes.h index 8dcd0b7..1b4dee8 100644 --- a/src/mpid/pamid/include/mpidi_datatypes.h +++ b/src/mpid/pamid/include/mpidi_datatypes.h @@ -158,6 +158,8 @@ enum MPIDI_Protocols_WinCtrl, MPIDI_Protocols_WinAccum, MPIDI_Protocols_RVZ_zerobyte, + MPIDI_Protocols_WinGetAccum, + MPIDI_Protocols_WinGetAccumAck, #ifdef DYNAMIC_TASKING MPIDI_Protocols_Dyntask, MPIDI_Protocols_Dyntask_disconnect, diff --git a/src/mpid/pamid/include/mpidi_prototypes.h b/src/mpid/pamid/include/mpidi_prototypes.h index ec50f71..dfdd1b8 100644 --- a/src/mpid/pamid/include/mpidi_prototypes.h +++ b/src/mpid/pamid/include/mpidi_prototypes.h @@ -204,6 +204,25 @@ MPIDI_WinControlCB(pami_context_t context, pami_endpoint_t sender, pami_recv_t * recv); +void +MPIDI_WinGetAccumCB(pami_context_t context, + void * cookie, + const void * _control, + size_t size, + const void * sndbuf, + size_t sndlen, + pami_endpoint_t sender, + pami_recv_t * recv); +void +MPIDI_WinGetAccumAckCB(pami_context_t context, + void * cookie, + const void * _control, + size_t size, + const void * sndbuf, + size_t sndlen, + pami_endpoint_t sender, + pami_recv_t * recv); + /** \brief Helper function to complete a rendevous transfer */ pami_result_t MPIDI_RendezvousTransfer(pami_context_t context, void* rreq); pami_result_t MPIDI_RendezvousTransfer_SyncAck(pami_context_t context, void* rreq); diff --git a/src/mpid/pamid/src/misc/mpid_unimpl.c b/src/mpid/pamid/src/misc/mpid_unimpl.c index cef7096..03e9783 100644 --- a/src/mpid/pamid/src/misc/mpid_unimpl.c +++ b/src/mpid/pamid/src/misc/mpid_unimpl.c @@ -161,12 +161,4 @@ int MPID_Rget(void *origin_addr, int origin_count, return 0; } -int MPID_Get_accumulate(const void *origin_addr, int origin_count, - MPI_Datatype origin_datatype, void *result_addr, int result_count, - MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, - int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win *win) -{ - MPID_abort(); - return 0; -} diff --git a/src/mpid/pamid/src/mpid_init.c b/src/mpid/pamid/src/mpid_init.c index 445b33d..5809ec1 100644 --- a/src/mpid/pamid/src/mpid_init.c +++ b/src/mpid/pamid/src/mpid_init.c @@ -151,6 +151,8 @@ static struct struct protocol_t WinCtrl; struct protocol_t WinAccum; struct protocol_t RVZ_zerobyte; + struct protocol_t WinGetAccum; + struct protocol_t WinGetAccumAck; #ifdef DYNAMIC_TASKING struct protocol_t Dyntask; struct protocol_t Dyntask_disconnect; @@ -252,6 +254,26 @@ static struct }, .immediate_min = sizeof(MPIDI_MsgEnvelope), }, + .WinGetAccum = { + .func = MPIDI_WinGetAccumCB, + .dispatch = MPIDI_Protocols_WinGetAccum, + .options = { + .consistency = PAMI_HINT_ENABLE, + .long_header = PAMI_HINT_DISABLE, + .recv_immediate = PAMI_HINT_DISABLE, + }, + .immediate_min = sizeof(MPIDI_Win_GetAccMsgInfo), + }, + .WinGetAccumAck = { + .func = MPIDI_WinGetAccumAckCB, + .dispatch = MPIDI_Protocols_WinGetAccumAck, + .options = { + .consistency = PAMI_HINT_ENABLE, + .long_header = PAMI_HINT_DISABLE, + .recv_immediate = PAMI_HINT_DISABLE, + }, + .immediate_min = sizeof(MPIDI_Win_GetAccMsgInfo), + }, #ifdef DYNAMIC_TASKING .Dyntask = { .func = MPIDI_Recvfrom_remote_world, @@ -802,6 +824,8 @@ MPIDI_PAMI_dispath_init() MPIDI_PAMI_dispath_set(MPIDI_Protocols_WinCtrl, &proto_list.WinCtrl, NULL); MPIDI_PAMI_dispath_set(MPIDI_Protocols_WinAccum, &proto_list.WinAccum, NULL); MPIDI_PAMI_dispath_set(MPIDI_Protocols_RVZ_zerobyte, &proto_list.RVZ_zerobyte, NULL); + MPIDI_PAMI_dispath_set(MPIDI_Protocols_WinGetAccum, &proto_list.WinGetAccum, NULL); + MPIDI_PAMI_dispath_set(MPIDI_Protocols_WinGetAccumAck, &proto_list.WinGetAccumAck, NULL); #ifdef DYNAMIC_TASKING MPIDI_PAMI_dispath_set(MPIDI_Protocols_Dyntask, &proto_list.Dyntask, NULL); MPIDI_PAMI_dispath_set(MPIDI_Protocols_Dyntask_disconnect, &proto_list.Dyntask_disconnect, NULL); diff --git a/src/mpid/pamid/src/onesided/Makefile.mk b/src/mpid/pamid/src/onesided/Makefile.mk index 843f06d..901e6ab 100644 --- a/src/mpid/pamid/src/onesided/Makefile.mk +++ b/src/mpid/pamid/src/onesided/Makefile.mk @@ -28,6 +28,7 @@ noinst_HEADERS += \ lib_lib@MPILIBNAME@_la_SOURCES += \ src/mpid/pamid/src/onesided/mpid_1s.c \ src/mpid/pamid/src/onesided/mpid_win_accumulate.c \ + src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c \ src/mpid/pamid/src/onesided/mpid_win_create.c \ src/mpid/pamid/src/onesided/mpid_win_fence.c \ src/mpid/pamid/src/onesided/mpid_win_free.c \ diff --git a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c index c2878d8..ecdcf14 100644 --- a/src/mpid/pamid/src/onesided/mpid_win_accumulate.c +++ b/src/mpid/pamid/src/onesided/mpid_win_accumulate.c @@ -95,7 +95,7 @@ MPIDI_Accumulate(pami_context_t context, ++sync->started; - params.send.header.iov_base = &req->accum_headers[req->state.index]; + params.send.header.iov_base = &(((MPIDI_Win_MsgInfo *)req->accum_headers)[req->state.index]); params.send.data.iov_len = req->target.dt.map[req->state.index].DLOOP_VECTOR_LEN; params.send.data.iov_base = req->buffer + req->state.local_offset; diff --git a/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c new file mode 100644 index 0000000..085f3ce --- /dev/null +++ b/src/mpid/pamid/src/onesided/mpid_win_get_accumulate.c @@ -0,0 +1,517 @@ +/* begin_generated_IBM_copyright_prolog */ +/* */ +/* This is an automatically generated copyright prolog. */ +/* After initializing, DO NOT MODIFY OR MOVE */ +/* --------------------------------------------------------------- */ +/* Licensed Materials - Property of IBM */ +/* Blue Gene/Q 5765-PER 5765-PRP */ +/* */ +/* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */ +/* US Government Users Restricted Rights - */ +/* Use, duplication, or disclosure restricted */ +/* by GSA ADP Schedule Contract with IBM Corp. */ +/* */ +/* --------------------------------------------------------------- */ +/* */ +/* end_generated_IBM_copyright_prolog */ +/* (C)Copyright IBM Corp. 2007, 2011 */ +/** + * \file src/onesided/mpid_win_accumulate.c + * \brief ??? + */ +#include "mpidi_onesided.h" + +static void +MPIDI_Win_GetAccSendAckDoneCB(pami_context_t context, + void * _buf, + pami_result_t result) +{ + MPIU_Free (_buf); +} + +static void +MPIDI_Win_GetAccumSendAck(pami_context_t context, + void * _info, + pami_result_t result) +{ + MPIDI_Win_GetAccMsgInfo *msginfo = (MPIDI_Win_GetAccMsgInfo *) _info; + pami_result_t rc = PAMI_SUCCESS; + + //Copy from msginfo->addr to a contiguous buffer + MPIDI_Datatype result_dt; + char *buffer = NULL; + + MPIDI_Win_datatype_basic(msginfo->result_count, + msginfo->result_datatype, + &result_dt); + + int use_map = 0; + buffer = MPIU_Malloc(result_dt.size); + if (result_dt.contig) + memcpy(buffer, (msginfo->addr + result_dt.true_lb), result_dt.size); + else + { + use_map = 1; + MPID_assert(buffer != NULL); + + int mpi_errno = 0; + mpi_errno = MPIR_Localcopy(msginfo->addr, + msginfo->count, + msginfo->type, + buffer, + result_dt.size, + MPI_CHAR); + MPID_assert(mpi_errno == MPI_SUCCESS); + } + + //Schedule sends to source to result buffer and trigger completion + //callback there + pami_send_t params = { + .send = { + .header = { + .iov_len = sizeof(MPIDI_Win_GetAccMsgInfo), + }, + .dispatch = MPIDI_Protocols_WinGetAccumAck, + .dest = msginfo->src_endpoint, + }, + .events = { + .cookie = buffer, + .local_fn = MPIDI_Win_GetAccSendAckDoneCB, //cleanup buffer + }, + }; + + int index = 0; + size_t local_offset = 0; + //Set the map + MPIDI_Win_datatype_map(&result_dt); + MPID_assert(result_dt.num_contig == msginfo->result_num_contig); + + while (index < result_dt.num_contig) { + params.send.header.iov_base = msginfo; + params.send.data.iov_len = result_dt.map[index].DLOOP_VECTOR_LEN; + params.send.data.iov_base = buffer + local_offset; + + rc = PAMI_Send(context, &params); + MPID_assert(rc == PAMI_SUCCESS); + local_offset += params.send.data.iov_len; + ++index; + } + + /** Review PAMI_Send semantics and consider moving the free calls to + the completion callback*/ + //free msginfo + MPIU_Free(msginfo); + + if (use_map && result_dt.map != &result_dt.__map) + MPIU_Free (result_dt.map); +} + +void +MPIDI_WinGetAccumCB(pami_context_t context, + void * cookie, + const void * _msginfo, + size_t msginfo_size, + const void * sndbuf, + size_t sndlen, + pami_endpoint_t sender, + pami_recv_t * recv) +{ + MPID_assert(recv != NULL); + MPID_assert(sndbuf == NULL); + MPID_assert(msginfo_size == sizeof(MPIDI_Win_GetAccMsgInfo)); + MPID_assert(_msginfo != NULL); + MPIDI_Win_GetAccMsgInfo * msginfo = (MPIDI_Win_GetAccMsgInfo *) + MPIU_Malloc(sizeof(MPIDI_Win_GetAccMsgInfo)); + + *msginfo = *(MPIDI_Win_GetAccMsgInfo *)_msginfo; + msginfo->src_endpoint = sender; + + int null=0; + pami_type_t pami_type; + pami_data_function pami_op; + MPI_Op op = msginfo->op; + + MPIDI_Datatype_to_pami(msginfo->type, &pami_type, op, &pami_op, &null); + + recv->addr = msginfo->addr; + recv->type = pami_type; + recv->offset = 0; + recv->data_fn = pami_op; + recv->data_cookie = NULL; + recv->local_fn = NULL; + recv->cookie = NULL; + + if (msginfo->counter == 0) + //We will now allocate a tempbuf, copy local contents and start a + //send + MPIDI_Win_GetAccumSendAck (context, msginfo, PAMI_SUCCESS); + else + MPIU_Free(msginfo); +} + +static void +MPIDI_Win_GetAccDoneCB(pami_context_t context, + void * cookie, + pami_result_t result) +{ + MPIDI_Win_request *req = (MPIDI_Win_request*)cookie; + ++req->win->mpid.sync.complete; + ++req->origin.completed; + + if (req->origin.completed == + (req->result_num_contig + req->target.dt.num_contig)) + { + if (req->buffer_free) + MPIU_Free(req->buffer); + if (req->accum_headers) + MPIU_Free(req->accum_headers); + MPIU_Free (req); + } + MPIDI_Progress_signal(); +} + +void +MPIDI_WinGetAccumAckCB(pami_context_t context, + void * cookie, + const void * _msginfo, + size_t msginfo_size, + const void * sndbuf, + size_t sndlen, + pami_endpoint_t sender, + pami_recv_t * recv) +{ + MPID_assert(recv != NULL); + MPID_assert(sndbuf == NULL); + MPID_assert(msginfo_size == sizeof(MPIDI_Win_GetAccMsgInfo)); + MPID_assert(_msginfo != NULL); + const MPIDI_Win_GetAccMsgInfo * msginfo =(const MPIDI_Win_GetAccMsgInfo *)_msginfo; + + int null=0; + pami_type_t pami_type; + pami_data_function pami_op; + MPI_Op op = msginfo->op; + + MPIDI_Datatype_to_pami(msginfo->result_datatype, &pami_type, op, &pami_op, &null); + + recv->addr = msginfo->result_addr; + recv->type = pami_type; + recv->offset = 0; + recv->data_fn = PAMI_DATA_COPY; + recv->data_cookie = NULL; + recv->local_fn = MPIDI_Win_GetAccDoneCB; + recv->cookie = msginfo->req; +} + +static pami_result_t +MPIDI_Get_accumulate(pami_context_t context, + void * _req) +{ + MPIDI_Win_request *req = (MPIDI_Win_request*)_req; + pami_result_t rc; + void *map; + + pami_send_t params = { + .send = { + .header = { + .iov_len = sizeof(MPIDI_Win_GetAccMsgInfo), + }, + .dispatch = MPIDI_Protocols_WinGetAccum, + .dest = req->dest, + }, + .events = { + .cookie = req, + .remote_fn = MPIDI_Win_GetAccDoneCB, //When all accumulates have + //completed remotely + //complete accumulate + }, + }; + + struct MPIDI_Win_sync* sync = &req->win->mpid.sync; + TRACE_ERR("Start index=%u/%d l-addr=%p r-base=%p r-offset=%zu (sync->started=%u sync->complete=%u)\n", + req->state.index, req->target.dt.num_contig, req->buffer, req->win->mpid.info[req->target.rank].base_addr, req->offset, sync->started, sync->complete); + while (req->state.index < req->target.dt.num_contig) { + if (sync->started > sync->complete + MPIDI_Process.rma_pending) + { + TRACE_ERR("Bailing out; index=%u/%d sync->started=%u sync->complete=%u\n", + req->state.index, req->target.dt.num_contig, sync->started, sync->complete); + return PAMI_EAGAIN; + } + ++sync->started; + + params.send.header.iov_base = &(((MPIDI_Win_GetAccMsgInfo *)req->accum_headers)[req->state.index]); + params.send.data.iov_len = req->target.dt.map[req->state.index].DLOOP_VECTOR_LEN; + params.send.data.iov_base = req->buffer + req->state.local_offset; + + if (req->target.dt.num_contig - req->state.index == 1) { + map=NULL; + if (req->target.dt.map != &req->target.dt.__map) { + map=(void *) req->target.dt.map; + } + + rc = PAMI_Send(context, &params); + MPID_assert(rc == PAMI_SUCCESS); + if (map) + MPIU_Free(map); + return PAMI_SUCCESS; + } else { + rc = PAMI_Send(context, &params); + MPID_assert(rc == PAMI_SUCCESS); + req->state.local_offset += params.send.data.iov_len; + ++req->state.index; + } + } + + MPIDI_Win_datatype_unmap(&req->target.dt); + + return PAMI_SUCCESS; +} + + +/** + * \brief MPI-PAMI glue for MPI_GET_ACCUMULATE function + * + * According to the MPI Specification: + * + * Each datatype argument must be a predefined datatype or + * a derived datatype, where all basic components are of the + * same predefined datatype. Both datatype arguments must be + * constructed from the same predefined datatype. + * + * \param[in] origin_addr Source buffer + * \param[in] origin_count Number of datatype elements + * \param[in] origin_datatype Source datatype + * \param[in] target_rank Destination rank (target) + * \param[in] target_disp Displacement factor in target buffer + * \param[in] target_count Number of target datatype elements + * \param[in] target_datatype Destination datatype + * \param[in] op Operand to perform + * \param[in] win Window + * \return MPI_SUCCESS + */ +#undef FUNCNAME +#define FUNCNAME MPID_Get_accumulate +#undef FCNAME +#define FCNAME MPIU_QUOTE(FUNCNAME) +int +MPID_Get_accumulate(const void * origin_addr, + int origin_count, + MPI_Datatype origin_datatype, + void * result_addr, + int result_count, + MPI_Datatype result_datatype, + int target_rank, + MPI_Aint target_disp, + int target_count, + MPI_Datatype target_datatype, + MPI_Op op, + MPID_Win *win) +{ + int mpi_errno = MPI_SUCCESS; + + if (op == MPI_NO_OP) {//we just need to fetch data + mpi_errno = MPID_Get(result_addr, + result_count, + result_datatype, + target_rank, + target_disp, + target_count, + target_datatype, + win); + return mpi_errno; + } + + MPIDI_Win_request *req; + req = MPIU_Calloc0(1, MPIDI_Win_request); + req->win = win; + req->type = MPIDI_WIN_REQUEST_ACCUMULATE, + req->offset = target_disp*win->mpid.info[target_rank].disp_unit; +#ifdef __BGQ__ + /* PAMI limitation as it doesnt permit VA of 0 to be passed into + * memregion create, so we must pass base_va of heap computed from + * an SPI call instead. So the target offset must be adjusted */ + if (req->win->create_flavor == MPI_WIN_FLAVOR_DYNAMIC) + req->offset -= (size_t)req->win->mpid.info[target_rank].base_addr; +#endif + + if(win->mpid.sync.origin_epoch_type == win->mpid.sync.target_epoch_type && + win->mpid.sync.origin_epoch_type == MPID_EPOTYPE_REFENCE){ + win->mpid.sync.origin_epoch_type = MPID_EPOTYPE_FENCE; + win->mpid.sync.target_epoch_type = MPID_EPOTYPE_FENCE; + } + + if(win->mpid.sync.origin_epoch_type == MPID_EPOTYPE_NONE || + win->mpid.sync.origin_epoch_type == MPID_EPOTYPE_POST){ + MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_SYNC, + return mpi_errno, "**rmasync"); + } + + req->offset = target_disp * win->mpid.info[target_rank].disp_unit; + + if (origin_datatype == MPI_DOUBLE_INT) { + MPIDI_Win_datatype_basic(origin_count*2, + MPI_DOUBLE, + &req->origin.dt); + MPIDI_Win_datatype_basic(target_count*2, + MPI_DOUBLE, + &req->target.dt); + } + else if (origin_datatype == MPI_LONG_DOUBLE_INT) + { + MPIDI_Win_datatype_basic(origin_count*2, + MPI_LONG_DOUBLE, + &req->origin.dt); + MPIDI_Win_datatype_basic(target_count*2, + MPI_LONG_DOUBLE, + &req->target.dt); + } + else if (origin_datatype == MPI_LONG_INT) + { + MPIDI_Win_datatype_basic(origin_count*2, + MPI_LONG, + &req->origin.dt); + MPIDI_Win_datatype_basic(target_count*2, + MPI_LONG, + &req->target.dt); + } + else if (origin_datatype == MPI_SHORT_INT) + { + MPIDI_Win_datatype_basic(origin_count*2, + MPI_INT, + &req->origin.dt); + MPIDI_Win_datatype_basic(target_count*2, + MPI_INT, + &req->target.dt); + } + else + { + MPIDI_Win_datatype_basic(origin_count, + origin_datatype, + &req->origin.dt); + MPIDI_Win_datatype_basic(target_count, + target_datatype, + &req->target.dt); + } + + MPID_assert(req->origin.dt.size == req->target.dt.size); + + if ( (req->origin.dt.size == 0) || + (target_rank == MPI_PROC_NULL)) + { + MPIU_Free (req); + return MPI_SUCCESS; + } + + req->target.rank = target_rank; + + + if (req->origin.dt.contig) + { + req->buffer_free = 0; + req->buffer = (char*)origin_addr + req->origin.dt.true_lb; + } + else + { + req->buffer_free = 1; + req->buffer = MPIU_Malloc(req->origin.dt.size); + MPID_assert(req->buffer != NULL); + + int mpi_errno = 0; + mpi_errno = MPIR_Localcopy(origin_addr, + origin_count, + origin_datatype, + req->buffer, + req->origin.dt.size, + MPI_CHAR); + MPID_assert(mpi_errno == MPI_SUCCESS); + } + + pami_result_t rc; + pami_task_t task = MPID_VCR_GET_LPID(win->comm_ptr->vcr, target_rank); + if (win->mpid.sync.origin_epoch_type == MPID_EPOTYPE_START && + !MPIDI_valid_group_rank(task, win->mpid.sync.sc.group)) + { + MPIU_ERR_SETANDSTMT(mpi_errno, MPI_ERR_RMA_SYNC, + return mpi_errno, "**rmasync"); + } + + rc = PAMI_Endpoint_create(MPIDI_Client, task, 0, &req->dest); + MPID_assert(rc == PAMI_SUCCESS); + + + MPIDI_Win_datatype_map(&req->target.dt); + MPIDI_Datatype result_dt; + MPIDI_Win_datatype_basic(result_count, result_datatype, &result_dt); + req->result_num_contig = 1; + if (!result_dt.contig) + req->result_num_contig =result_dt.pointer->max_contig_blocks*result_count+1; + //We wait for #messages depending on target and result_datatype + win->mpid.sync.total += (req->result_num_contig + req->target.dt.num_contig); + + { + MPI_Datatype basic_type = MPI_DATATYPE_NULL; + MPID_Datatype_get_basic_type(origin_datatype, basic_type); + /* MPID_Datatype_get_basic_type() doesn't handle the struct types */ + if ((origin_datatype == MPI_FLOAT_INT) || + (origin_datatype == MPI_DOUBLE_INT) || + (origin_datatype == MPI_LONG_INT) || + (origin_datatype == MPI_SHORT_INT) || + (origin_datatype == MPI_LONG_DOUBLE_INT)) + { + MPID_assert(basic_type == MPI_DATATYPE_NULL); + basic_type = origin_datatype; + } + MPID_assert(basic_type != MPI_DATATYPE_NULL); + + MPI_Datatype result_basic_type = MPI_DATATYPE_NULL; + MPID_Datatype_get_basic_type(result_datatype, result_basic_type); + /* MPID_Datatype_get_basic_type() doesn't handle the struct types */ + if ((result_datatype == MPI_FLOAT_INT) || + (result_datatype == MPI_DOUBLE_INT) || + (result_datatype == MPI_LONG_INT) || + (result_datatype == MPI_SHORT_INT) || + (result_datatype == MPI_LONG_DOUBLE_INT)) + { + MPID_assert(result_basic_type == MPI_DATATYPE_NULL); + result_basic_type = result_datatype; + } + MPID_assert(result_basic_type != MPI_DATATYPE_NULL); + + unsigned index; + MPIDI_Win_GetAccMsgInfo * headers = MPIU_Calloc0(req->target.dt.num_contig, MPIDI_Win_GetAccMsgInfo); + req->accum_headers = headers; + for (index=0; index < req->target.dt.num_contig; ++index) { + headers[index].addr = win->mpid.info[target_rank].base_addr + req->offset + + (size_t)req->target.dt.map[index].DLOOP_VECTOR_BUF; + headers[index].req = req; + headers[index].win = win; + headers[index].type = basic_type; + headers[index].op = op; + headers[index].count = target_count; + headers[index].counter = index; + headers[index].num_contig = req->target.dt.num_contig; + headers[index].request = req; + headers[index].result_addr = result_addr; + headers[index].result_count = result_count; + headers[index].result_datatype = result_basic_type; + headers[index].result_num_contig= req->result_num_contig; + } + + } + + /* The pamid one-sided design requires context post in order to handle the + * case where the number of pending rma operation exceeds the + * 'PAMID_RMA_PENDING' threshold. When there are too many pending requests the + * work function remains on the context post queue (by returning PAMI_EAGAIN) + * so that the next time the context is advanced the work function will be + * invoked again. + * + * TODO - When context post is not required it would be better to attempt a + * direct context operation and then fail over to using context post if + * the rma pending threshold has been reached. This would result in + * better latency for one-sided operations. + */ + PAMI_Context_post(MPIDI_Context[0], &req->post_request, MPIDI_Get_accumulate, req); + +fn_fail: + return mpi_errno; +} diff --git a/src/mpid/pamid/src/onesided/mpidi_onesided.h b/src/mpid/pamid/src/onesided/mpidi_onesided.h index 8fa896b..e29b1e7 100644 --- a/src/mpid/pamid/src/onesided/mpidi_onesided.h +++ b/src/mpid/pamid/src/onesided/mpidi_onesided.h @@ -115,9 +115,27 @@ typedef struct size_t len; /* length of the send data */ } MPIDI_Win_MsgInfo; +typedef struct +{ + void * addr; + void * req; + MPID_Win * win; + MPI_Datatype type; + MPI_Op op; + int count; + int counter; + int num_contig; + void * request; + void * result_addr; + int result_count; + MPI_Datatype result_datatype; + int result_num_contig; + pami_endpoint_t src_endpoint; +} MPIDI_Win_GetAccMsgInfo; + /** \todo make sure that the extra fields are removed */ -typedef struct +typedef struct _mpidi_win_request { MPID_Win *win; MPIDI_Win_requesttype_t type; @@ -131,7 +149,7 @@ typedef struct size_t local_offset; } state; - MPIDI_Win_MsgInfo * accum_headers; + void * accum_headers; struct { @@ -153,9 +171,15 @@ typedef struct MPIDI_Datatype dt; } target; - void *buffer; void *user_buffer; uint32_t buffer_free; + void *buffer; + struct _mpidi_win_request *next; + void * compare_addr; + void * result_addr; + MPI_Op op; + int result_num_contig; + } MPIDI_Win_request; MPIDI_Win_request zero_req; /* used for init. request structure to 0 */ ----------------------------------------------------------------------- hooks/post-receive -- MPICH primary repository

1 0

[mpich] MPICH primary repository branch, master, updated. v3.1rc2-74-g356762d
by noreply＠mpich.org 18 Dec '13

18 Dec '13

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "MPICH primary repository". The branch, master has been updated via 356762dbfb82359a673598c1e3577111eb4def99 (commit) via b34fc532842ef51a9f240a6220227bd36df3d3c4 (commit) via 0c7ce9e24a3731e3d2275170de6e3aaa9a58ee1c (commit) from d2b786dc15ec29c53330cf44afe8ef250e302e0f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- http://git.mpich.org/mpich.git/commitdiff/356762dbfb82359a673598c1e3577111e… commit 356762dbfb82359a673598c1e3577111eb4def99 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Wed Dec 11 15:19:17 2013 -0600 ROMIO standalone: check for large-count routines the MPI_Type_size_x and MPI_Status_set_elements_x might not be available on older MPI implementations. Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpi/romio/configure.ac b/src/mpi/romio/configure.ac index c5ca7ee..8f05dfa 100644 --- a/src/mpi/romio/configure.ac +++ b/src/mpi/romio/configure.ac @@ -1547,7 +1547,9 @@ elif test $FROM_MPICH = yes ; then DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST 1" AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined]) AC_DEFINE(HAVE_MPIX_H, 1, []) - AC_DEFINE(HAVE_MPIIO_CONST, const, Set if MPI-IO prototypes use const qualifier), + AC_DEFINE(HAVE_MPIIO_CONST, const, Set if MPI-IO prototypes use const qualifier) + AC_DEFINE(HAVE_MPI_TYPE_SIZE_X, 1, [Define if MPI library provides MPI_TYPE_SIZE_X]) + AC_DEFINE(HAVE_MPI_STATUS_SET_ELEMENTS_X, 1, [Define if MPI library provides MPI_STATUS_SET_ELEMENTS_X]) fi # # @@ -1562,6 +1564,7 @@ if test $WITHIN_KNOWN_MPI_IMPL = no ; then AC_CHECK_TYPE(MPI_Count,[],[AC_DEFINE_UNQUOTED([MPI_Count],[MPI_Aint],[Define to "MPI_Aint" if MPI does not provide MPI_Count]) ]) PAC_TEST_NEEDS_CONST AC_CHECK_DECLS([MPI_COMBINER_HINDEXED_BLOCK], [], [], [[#include <mpi.h>]]) + AC_CHECK_FUNCS(MPI_Type_size_x MPI_Status_set_elements_x) fi # if test -z "$TEST_CC" ; then diff --git a/src/mpi/romio/mpi-io/glue/Makefile.mk b/src/mpi/romio/mpi-io/glue/Makefile.mk index ca3bafe..05954a1 100644 --- a/src/mpi/romio/mpi-io/glue/Makefile.mk +++ b/src/mpi/romio/mpi-io/glue/Makefile.mk @@ -7,3 +7,8 @@ include $(top_srcdir)/mpi-io/glue/default/Makefile.mk include $(top_srcdir)/mpi-io/glue/mpich/Makefile.mk + +if !BUILD_ROMIO_EMBEDDED +romio_other_sources += \ + mpi-io/glue/large_count.c +endif diff --git a/src/mpi/romio/mpi-io/glue/large_count.c b/src/mpi/romio/mpi-io/glue/large_count.c new file mode 100644 index 0000000..ee31e2b --- /dev/null +++ b/src/mpi/romio/mpi-io/glue/large_count.c @@ -0,0 +1,26 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + */ + +#include "mpioimpl.h" + +#ifndef HAVE_MPI_TYPE_SIZE_X +int MPI_Type_size_x(MPI_Datatype datatype, MPI_Count *size) +{ + int size_int, ret; + ret = MPI_Type_size(datatype, &size_int); + *size = size_int; + return ret; +} +#endif + +#ifndef HAVE_MPI_STATUS_SET_ELEMENTS_X +int MPI_Status_set_elements_x(MPI_Status *status, MPI_Datatype datatype, + MPI_Count count) +{ + int count_int = (int) count; + return MPI_Status_set_elements(status, datatype, count_int); +} +#endif http://git.mpich.org/mpich.git/commitdiff/b34fc532842ef51a9f240a6220227bd36… commit b34fc532842ef51a9f240a6220227bd36df3d3c4 Author: Rob Latham <robl(a)mcs.anl.gov> Date: Wed Dec 11 15:17:09 2013 -0600 ROMIO standalone: fix up link-time errors - zap the TRACE_ERR routine - use ADIOI_MAX instead of pami-defined MAX - find proper header file for Kernel_GetPersonality Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_aggrs.c b/src/mpi/romio/adio/ad_bg/ad_bg_aggrs.c index d6eeff4..dea444e 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_aggrs.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_aggrs.c @@ -28,6 +28,10 @@ #define AGG_DEBUG 1 #endif +#ifndef TRACE_ERR +# define TRACE_ERR(format...) +#endif + /* Comments copied from common: * This file contains four functions: * diff --git a/src/mpi/romio/adio/ad_bg/ad_bg_pset.c b/src/mpi/romio/adio/ad_bg/ad_bg_pset.c index ea6774f..3e842da 100644 --- a/src/mpi/romio/adio/ad_bg/ad_bg_pset.c +++ b/src/mpi/romio/adio/ad_bg/ad_bg_pset.c @@ -16,12 +16,17 @@ #include <stdlib.h> #include "ad_bg.h" #include "ad_bg_pset.h" +#include <spi/include/kernel/process.h> #include <firmware/include/personality.h> #ifdef HAVE_MPIX_H #include <mpix.h> #endif +#ifndef TRACE_ERR +# define TRACE_ERR(fmt...) +#endif + ADIOI_BG_ProcInfo_t * ADIOI_BG_ProcInfo_new() { @@ -258,7 +263,7 @@ ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, if(conf->nAggrs <=0) conf->nAggrs = bgmpio_bg_nagg_pset; if(conf->ioMinSize <= conf->nAggrs) - conf->nAggrs = MAX(1,conf->ioMinSize-1); /* not including bridge itself */ + conf->nAggrs = ADIOI_MAX(1,conf->ioMinSize-1); /* not including bridge itself */ /* if(conf->nAggrs > conf->numBridgeRanks) conf->nAggrs = conf->numBridgeRanks; */ http://git.mpich.org/mpich.git/commitdiff/0c7ce9e24a3731e3d2275170de6e3aaa9… commit 0c7ce9e24a3731e3d2275170de6e3aaa9a58ee1c Author: Rob Latham <robl(a)mcs.anl.gov> Date: Wed Dec 11 14:38:36 2013 -0600 ROMIO standalone: actually install library the "install these bits" part had been removed in the great makefile shakeup of late 2011. now "make install" installs a 'libromio' library Signed-off-by: Ken Raffenetti <raffenet(a)mcs.anl.gov> diff --git a/src/mpi/romio/Makefile.am b/src/mpi/romio/Makefile.am index 1626304..e27323f 100644 --- a/src/mpi/romio/Makefile.am +++ b/src/mpi/romio/Makefile.am @@ -86,7 +86,12 @@ libpromio_la_CPPFLAGS = $(AM_CPPFLAGS) -DMPIO_BUILD_PROFILING endif BUILD_PROFILING_LIB else !BUILD_ROMIO_EMBEDDED -## TODO build a libromio.la (non-convenience) and possibly a libglue.la or something? +lib_LTLIBRARIES = libromio.la +libromio_la_SOURCES = $(romio_mpi_sources) $(romio_other_sources) $(glue_sources) +if BUILD_PROFILING_LIB +libpromio_la_SOURCES = $(romio_mpi_sources) +libpromio_la_CPPFLAGS = $(AM_CPPFLAGS) -DMPIO_BUILD_PROFILING +endif BUILD_PROFILING_LIB endif diff --git a/src/mpi/romio/configure.ac b/src/mpi/romio/configure.ac index 726237f..c5ca7ee 100644 --- a/src/mpi/romio/configure.ac +++ b/src/mpi/romio/configure.ac @@ -632,8 +632,7 @@ else fi AC_SUBST(HAVE_WEAK_SYMBOLS) -# FIXME we only build ROMIO in embedded mode for now -AM_CONDITIONAL([BUILD_ROMIO_EMBEDDED],[true]) +AM_CONDITIONAL([BUILD_ROMIO_EMBEDDED],[test "$WITHIN_KNOWN_MPI_IMPL" = "yes" ]) # FIXME need to get this right for non-MPICH builds AM_CONDITIONAL([BUILD_MPIO_REQUEST],[false]) # FIXME need to get this right for non-MPICH builds ----------------------------------------------------------------------- Summary of changes: src/mpi/romio/Makefile.am | 7 ++++++- src/mpi/romio/adio/ad_bg/ad_bg_aggrs.c | 4 ++++ src/mpi/romio/adio/ad_bg/ad_bg_pset.c | 7 ++++++- src/mpi/romio/configure.ac | 8 +++++--- src/mpi/romio/mpi-io/glue/Makefile.mk | 5 +++++ src/mpi/romio/mpi-io/glue/large_count.c | 26 ++++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 src/mpi/romio/mpi-io/glue/large_count.c hooks/post-receive -- MPICH primary repository

1 0