Callbacks provided in MPI_Comm_create_keyval are not called
I am reading Using MPI and trying to execute the code itself. Chapter 6.2 contains non-blocking broadcast code. I tried to run my own callbacks instead of MPI_NULL_COPY_FN
or MPI_NULL_DELETE_FN
. Here is my code, it is very similar to the code in the book, but the callbacks will not be called. I do not know why. When compiled with -Wall
no warnings or errors. Could you help me please? Many thanks.
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
static int ibcast_keyval=MPI_KEYVAL_INVALID;
typedef struct
{
MPI_Comm comm;
int ordering_tag;
} Ibcast_syshandle;
typedef struct
{
MPI_Request *req_array;
MPI_Status *stat_array;
int num_sends;
int num_recvs;
} Ibcast_handle;
int Ibcast_work(Ibcast_handle *handle)
{
if(handle->num_recvs==0)
MPI_Startall(handle->num_sends, handle->req_array);
else
MPI_Startall(handle->num_recvs, &(handle->req_array[handle->num_sends]));
return MPI_SUCCESS;
}
int Ibcast_copy(MPI_Comm oldcomm, int keyval, void *extra, void *attr_in, void *attr_out, int *flag)
{
Ibcast_syshandle *syshandle=(Ibcast_syshandle *)attr_in;
Ibcast_syshandle *new_syshandle;
printf("keyval=%d\n", keyval);
fflush(stdout);
if((keyval==MPI_KEYVAL_INVALID)||(keyval!=ibcast_keyval)||(syshandle==NULL))
return 1;
new_syshandle=(Ibcast_syshandle *)malloc(sizeof(Ibcast_syshandle));
new_syshandle->ordering_tag=0;
MPI_Comm_dup(syshandle->comm, &(new_syshandle->comm));
{
int rank;
MPI_Comm_rank(new_syshandle->comm, &rank);
printf("Ibcast_copy called from %d\n", rank);
fflush(stdout);
}
*(void **)attr_out=(void *)new_syshandle;
*flag=1;
return MPI_SUCCESS;
}
int Ibcast_delete(MPI_Comm comm, int keyval, void *attr_val, void *extra)
{
Ibcast_syshandle *syshandle=(Ibcast_syshandle *)attr_val;
{
int rank;
MPI_Comm_rank(syshandle->comm, &rank);
printf("Ibcast_delete called from %d\n", rank);
fflush(stdout);
}
if((keyval==MPI_KEYVAL_INVALID)||(keyval!=ibcast_keyval)||(syshandle==NULL))
return 1;
MPI_Comm_free(&(syshandle->comm));
free(syshandle);
return MPI_SUCCESS;
}
int Ibcast(void *buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm, Ibcast_handle **handle_out)
{
Ibcast_syshandle *syshandle;
Ibcast_handle *handle;
int flag, mask, relrank;
int retn, size, rank;
int req_no=0;
MPI_Comm_size(comm, &size);
MPI_Comm_rank(comm, &rank);
if(size==1)
{
(*handle_out)=NULL;
return MPI_SUCCESS;
}
if(ibcast_keyval==MPI_KEYVAL_INVALID)
// MPI_Keyval_create(MPI_NULL_COPY_FN, MPI_NULL_DELETE_FN, &ibcast_keyval, NULL);
MPI_Comm_create_keyval(Ibcast_copy, Ibcast_delete, &ibcast_keyval, NULL);
MPI_Comm_get_attr(comm, ibcast_keyval, (void **)&syshandle, &flag);
if(flag==0)
{
syshandle=(Ibcast_syshandle *)malloc(sizeof(Ibcast_syshandle));
syshandle->ordering_tag=0;
MPI_Comm_dup(comm, &(syshandle->comm));
MPI_Comm_set_attr(comm, ibcast_keyval, (void *)syshandle);
}
handle=(Ibcast_handle *)malloc(sizeof(Ibcast_handle));
handle->num_sends=handle->num_recvs=0;
mask=0x1;
relrank=(rank-root+size)%size;
while((mask&relrank)==0 && mask<size)
{
if((relrank|mask)<size)
++handle->num_sends;
mask<<=1;
}
if(mask<size)
++handle->num_recvs;
handle->req_array=(MPI_Request *)malloc(sizeof(MPI_Request)*(handle->num_sends+handle->num_recvs));
handle->stat_array=(MPI_Status *)malloc(sizeof(MPI_Status)*(handle->num_sends+handle->num_recvs));
mask=0x1;
relrank=(rank-root+size)%size;
while((mask&relrank)==0 && mask<size)
{
if((relrank|mask)<size)
MPI_Send_init(buf, count, datatype, ((relrank|mask)+root)%size, syshandle->ordering_tag, syshandle->comm, &(handle->req_array[req_no++]));
mask<<=1;
}
if(mask<size)
MPI_Recv_init(buf, count, datatype, ((relrank & (~mask))+root)%size, syshandle->ordering_tag, syshandle->comm, &(handle->req_array[req_no++]));
retn=Ibcast_work(handle);
++(syshandle->ordering_tag);
(*handle_out)=handle;
return retn;
}
int Ibcast_wait(Ibcast_handle **handle_out)
{
Ibcast_handle *handle=(*handle_out);
int retn, i;
if(handle==NULL)
return MPI_SUCCESS;
if(handle->num_recvs!=0)
{
MPI_Waitall(handle->num_recvs, &handle->req_array[handle->num_sends], &handle->stat_array[handle->num_sends]);
MPI_Startall(handle->num_sends, handle->req_array);
}
retn=MPI_Waitall(handle->num_sends, handle->req_array, handle->stat_array);
for(i=0; i<(handle->num_sends+handle->num_recvs);i++)
MPI_Request_free(&(handle->req_array[i]));
free(handle->req_array);
free(handle->stat_array);
free(handle);
*handle_out=NULL;
return retn;
}
int main( int argc, char *argv[] )
{
int buf1[10], buf2[20];
int rank, i;
Ibcast_handle *ibcast_handle_1, *ibcast_handle_2;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if (rank == 0) {
for (i=0; i<10; i++) buf1[i] = i;
for (i=0; i<20; i++) buf2[i] = -i;
}
Ibcast( buf1, 10, MPI_INT, 0, MPI_COMM_WORLD, &ibcast_handle_1 );
Ibcast( buf2, 20, MPI_INT, 0, MPI_COMM_WORLD, &ibcast_handle_2 );
Ibcast_wait( &ibcast_handle_1 );
Ibcast_wait( &ibcast_handle_2 );
for (i=0; i<10; i++) {
if (buf1[i] != i) printf( "buf1[%d] = %d on %d\n", i, buf1[i], rank );
}
for (i=0; i<20; i++) {
if (buf2[i] != -i) printf( "buf2[%d] = %d on %d\n", i, buf2[i], rank );
}
MPI_Finalize();
return 0;
}
source to share
Callback functions are for copying and deleting created attributes when duplicating or deleting a communicator, or just deleting an attribute. Callback functions are needed because attributes can be completely arbitrary.
So, here's a stripped-down version of your code that works (making such a minimal example is a useful way to both troubleshoot problems and get help on sites like SO):
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
static int ibcast_keyval;
int Ibcast_copy(MPI_Comm oldcomm, int keyval, void *extra, void *attr_in, void *attr_out, int *flag)
{
printf("In ibcast_copy: keyval = %d\n", keyval);
*flag = 1;
return MPI_SUCCESS;
}
int Ibcast_delete(MPI_Comm comm, int keyval, void *attr_val, void *extra)
{
printf("In ibcast_delete: keyval = %d\n", keyval);
return MPI_SUCCESS;
}
int main( int argc, char *argv[] )
{
int rank, i;
int attr=2;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Comm duped_comm;
MPI_Comm_create_keyval(Ibcast_copy, Ibcast_delete, &ibcast_keyval, NULL);
MPI_Comm_set_attr( MPI_COMM_WORLD, ibcast_keyval, &attr);
MPI_Comm_dup( MPI_COMM_WORLD, &duped_comm);
MPI_Comm_free( &duped_comm );
MPI_Comm_delete_attr( MPI_COMM_WORLD, ibcast_keyval );
MPI_Finalize();
return 0;
}
Here we create a key with callbacks, set the attribute corresponding to the key, and then duplicate MPI_COMM_WORLD (by calling the copy callback) and then release the duplicated communicator and remove the attribute from COMM_WORLD (by calling delete callback just :)
$ mpirun -np 1 ./comm-attr
In ibcast_copy: keyval = 10
In ibcast_delete: keyval = 10
In ibcast_delete: keyval = 10
In your code, you swap the communicator to Ibcast before setting the attribute, so the copy callback is never called (since there is nothing to copy). You can fix this part by setting the attribute before dup, but then there is another problem: you are calling dup and are free in callbacks, which is wrong; these functions should (indirectly) call callbacks, not vice versa.
source to share