MPI_Bcast inside an if statement

Question

1. Goal

Let's suppose we have world_size = 4 processors disposed as a topological mesh:

P0-P1
|  |  
P2-P3

where - represents mesh_r (mesh_rows) communicator and | represents mesh_c (mesh_columns) communicator, build through build_mesh procedure.

My goal is distribute P0 and P3 element called A_loc respectively to P1 and P2 (in general, from processors on the diagonal of the mesh to the ones on their rows).

I used an additional A_loc_add variable because I must not re-write A_loc variable during these distributions.

2. My code

int main(int argc, char *argv[])
{
    int process_rank, world_size;
    int mesh_rows, mesh_columns;
    int mesh_dimension = 2;
    int *process_coordinates;
    MPI_Comm mesh, mesh_r, mesh_c;
    int process_rank_mesh;
    int *A, *A_loc;
    int m, n, mloc, nloc;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &process_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &world_size);

    if (process_rank == 0) {
        m = n = 2;
    }

    MPI_Bcast(&m, 1, MPI_INT, 0, MPI_COMM_WORLD);
    MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD);
    A = fill_matrix(A, m, n);

    if (process_rank == 0) 
        mesh_rows = 2;

    if (is_divisible(world_size, mesh_rows))
        mesh_columns = world_size / mesh_rows;
    else {
        mesh_rows = 1;
        mesh_columns = world_size / mesh_rows;
    }
   
    MPI_Bcast(&mesh_rows, 1, MPI_INT, 0, MPI_COMM_WORLD);
    MPI_Bcast(&mesh_columns, 1, MPI_INT, 0, MPI_COMM_WORLD);

    process_coordinates = (int*) calloc(mesh_dimension, sizeof(int));
    build_mesh(&mesh, &mesh_r, &mesh_c, process_rank, world_size, mesh_rows, mesh_columns, process_coordinates);
    MPI_Comm_rank(mesh, &process_rank_mesh); 
 
    mloc = m / mesh_rows;
    nloc = m / mesh_columns;

    handle_errors(m, n, world_size, process_rank);

    A_loc = (int*) calloc(mloc * nloc, sizeof(int));
    distribute(A, A_loc, m, n, mloc, nloc, world_size, mesh_rows, mesh_columns);

    int *A_loc_add = (int*) calloc(mloc * nloc, sizeof(int));
    memcpy(A_loc_add, A_loc, sizeof(A_loc) * mloc);
    // process_rank is the processor rank in MPI_COMM_WORLD
    // so I choose P0 and P3 (the ones on the diagonal)
    // but they have rank 0 and 1 in mesh_r
    if (process_rank == 0)
        MPI_Bcast(A_loc_add, mloc * nloc, MPI_INT, 0, mesh_r);
    else if (process_rank == 3)
        MPI_Bcast(A_loc_add, mloc * nloc, MPI_INT, 1, mesh_r);

    MPI_Finalize();
    return 0;
}

void distribute(int *Mat, int *Mat_loc, int m, int n, int mloc, int nloc, int world_size, int mesh_rows, int mesh_columns)
{
    MPI_Datatype square_block;
    int stride = n;
    int count = mloc;
    int block_length = nloc;
    MPI_Type_vector(count, block_length, stride, MPI_INT, &square_block);
    MPI_Datatype square_block_resized;
    MPI_Type_create_resized(square_block, 0, sizeof(int), &square_block_resized);
    MPI_Type_commit(&square_block_resized);
    int *send_counts = (int*) calloc(world_size, sizeof(int));
    int *displs = (int*) calloc(world_size, sizeof(int));
    for (int i = 0; i < mesh_rows; i++) {
        for (int j = 0; j < mesh_columns; j++) {
            send_counts[i * mesh_columns + j] = 1;
            displs[i * mesh_columns + j] = i * n * block_length + j * block_length;
        }
    }
    
    MPI_Scatterv(Mat, send_counts, displs, square_block_resized, Mat_loc, mloc * nloc, MPI_INT, 0, MPI_COMM_WORLD);
}

void handle_errors(int m, int n, int world_size, int process_rank)
{
    if (process_rank == 0) {
        if (m != n) {
            perror("Not square matrices\n");
            MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
        }
        if (world_size != 4) {
            perror("World size must be 4\n");
            MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
        }
    }
}

bool is_divisible(int dividend, int divisor)
{
    return dividend % divisor == 0;
}

void build_mesh(MPI_Comm *mesh, MPI_Comm *mesh_r, MPI_Comm *mesh_c, int process_rank, int world_size,
    int mesh_rows, int mesh_columns, int *process_coordinates) 
{
    int mesh_dimension = 2;
    int *mesh_n_dimension;
    int mesh_reorder = 0;
    int *mesh_period;
    int *remain_dims = (int*) calloc(mesh_dimension, sizeof(int));
    mesh_n_dimension = (int*) calloc(mesh_dimension, sizeof(int));
    mesh_n_dimension[0] = mesh_rows;
    mesh_n_dimension[1] = mesh_columns;
    mesh_period = (int*) calloc(mesh_dimension, sizeof(int));
    mesh_period[0] = mesh_period[1] = 0;
    MPI_Cart_create(MPI_COMM_WORLD, mesh_dimension, mesh_n_dimension, mesh_period, mesh_reorder, mesh);
    MPI_Cart_coords(*mesh, process_rank, mesh_dimension, process_coordinates);
    remain_dims[0] = 0;  
    remain_dims[1] = 1;
    MPI_Cart_sub(*mesh, remain_dims, mesh_r);
    remain_dims[0] = 1;
    remain_dims[1] = 0;
    MPI_Cart_sub(*mesh, remain_dims, mesh_c);
}

int *fill_matrix(int *Mat, int m, int n)
{
    int k = 0;
    Mat = (int*) calloc(m * n, sizeof(int));
    for (int i = 0; i < m; i++)
        for (int j = 0; j < n; j++) 
            Mat[i * n + j] = ++k;
    return Mat;
}

3. Result

I get 1,2,3,4 instead of 1,1,4,4:

P0 A_loc_add:
1 

P1 A_loc_add:
2 (exp. 1)

P2 A_loc_add:
3 (exp. 4)

P3 A_loc_add:
4

Answer 1

Solved implementing a function such that f(0) = f(1) = 0 and 1 otherwise, as suggested by Gilles.