On Mon, Apr 10, 2017, at 10:58, Jeff Hammond wrote:

I do not understand why creating a wrapper function dynamically at run time creates a portability problem. Can you elaborate?

It needs to generate machine code at run time, like the example below. (A more sane way to do this would be through libffi or something.) #include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/mman.h> #include <unistd.h> #include <mpi.h> typedef void (*MpiUserFunctionPtr)(void *, void *, int *, MPI_Datatype *); // operations provided by user typedef void (*Operation)(double *, double *); _Thread_local Operation current_operation; // converts Operation to MPI_User_function void converter(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype) { current_operation(invec, inoutvec); } MpiUserFunctionPtr create_dynamic_wrapper(Operation operation) { MpiUserFunctionPtr converter_ptr = converter; Operation *current_operation_ptr = ¤t_operation; char *m = aligned_alloc(sysconf(_SC_PAGESIZE), 0x23); // unportable: assumes x86-64 memcpy(m + 0x00, "\x49\xba", 2); // movabs operation,%r10 memcpy(m + 0x02, &operation, 8); memcpy(m + 0x0a, "\x48\xb8", 2); // movabs ¤t_operation,%rax memcpy(m + 0x0c, ¤t_operation_ptr, 8); memcpy(m + 0x14, "\x4c\x89\x10", 3); // movabs %r10,(%rax) memcpy(m + 0x17, "\x48\xb8", 2); // movabs &converter,%rax memcpy(m + 0x19, &converter_ptr, 8); memcpy(m + 0x21, "\xff\xe0", 2); // jmpq *%rax mprotect(m, 0x23, PROT_READ | PROT_WRITE | PROT_EXEC); return (MpiUserFunctionPtr)m; } // === demonstration === // some test operations provided by the user (unaware of MPI) void add(double *invec, double *inoutvec) { *inoutvec += *invec; } void multiply(double *invec, double *inoutvec) { *inoutvec *= *invec; } int main(void) { int rank; MPI_Init(NULL, NULL); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Op op_add; MPI_Op_create(create_dynamic_wrapper(&add), 1, &op_add); MPI_Op op_multiply; MPI_Op_create(create_dynamic_wrapper(&multiply), 1, &op_multiply); double d = rank + 1; double r = 0.0; MPI_Reduce(&d, &r, 1, MPI_DOUBLE, op_add, 0, MPI_COMM_WORLD); if (rank == 0) { printf("∑(rank+1) = %.0f\n", r); } r = 1.0; MPI_Reduce(&d, &r, 1, MPI_DOUBLE, op_multiply, 0, MPI_COMM_WORLD); if (rank == 0) { printf("∏(rank+1) = %.0f\n", r); } MPI_Finalize(); }