/*
  This program performs a short test of MPI_BSEND in a
  multithreaded environment. 
  
  It starts a single receiver thread that expects 
  NUMSENDS messages and NUMSENDS sender threads, that
  use MPI_Bsend to send a message of size MSGSIZE
  to its right neigbour or rank 0 if (my_rank==comm_size-1), i.e.
    target_rank = (my_rank+1)%size .
    
  After all messages have been received, the
  receiver thread prints a message, the threads
  are joined into the main thread and the application
  terminates.
*/

#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <string.h>

#define NUMSENDS 32
#define BUFSIZE 100000
#define MSGSIZE 1024

int rank, size;


void* receiver(void *ptr){
  int k;
  char buf[MSGSIZE];

  for (k=0;k<NUMSENDS;k++)
    MPI_Recv( &buf, MSGSIZE, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
  
  printf("Received all\n");

  return NULL;
}


void* sender(void *ptr){
  char buffer[MSGSIZE];
  MPI_Bsend(&buffer, MSGSIZE, MPI_CHAR, (rank+1)%size, 0, MPI_COMM_WORLD);
  /* MPI_Send(&buffer, MSGSIZE, MPI_CHAR, (rank+1)%size, 0, MPI_COMM_WORLD); */

  return NULL;
}


int main(int argc, char *argv[]){

  int provided, i[2], k;
  void* buffer;
  MPI_Status  status;
  pthread_t receiver_thread, sender_thread;
  MPI_Comm communicator;
  int bs;

  MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

  if (provided != MPI_THREAD_MULTIPLE){
    printf("Error\n");
    MPI_Abort(911, MPI_COMM_WORLD);
  }
  
  bs = BUFSIZE;
  buffer = malloc(bs);
  MPI_Buffer_attach(buffer, BUFSIZE);
  
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  MPI_Comm_size(MPI_COMM_WORLD, &size);
  /*  MPI_Comm_dup(MPI_COMM_WORLD,&communicator); /* We do not use this communicator in this program, but
                                                 with this call, the problem appears more reliably.
                                                 If the MPI_Comm_dup() call is commented out, it is still
                                                 evident but does not appear that often (don't know why) */

  pthread_create(&receiver_thread, NULL, &receiver, NULL);

  for (k=0;k<NUMSENDS;k++)
    pthread_create(&sender_thread, NULL, &sender, NULL);

  pthread_join(receiver_thread, NULL);
  for (k=0;k<NUMSENDS;k++)
    pthread_join(sender_thread, NULL);
  
  printf("Threads joined...\n");

  MPI_Barrier(MPI_COMM_WORLD);

  printf("...on all ranks.\n");
  
  MPI_Comm_free(&communicator);
  MPI_Buffer_detach(&buffer, &bs);
  free(buffer);
  MPI_Finalize();
}