Search code examples
cparallel-processingmpidistributed-computingopenmpi

An error occurred in MPI_Scatterv (message truncated) while trying to distribute an array

1. Goal

Let's suppose we want to distribute a square mxn=4x4=16 array among 4 processors placed in a topological mesh/grid called mesh:

P0-P1
|  |  
P2-P3

where - represents mesh_r (mesh_rows) communicator and | represents mesh_c (mesh_columns) communicator, build through build_mesh procedure. Let's suppose m be a multiple of number of processors world_size. If

  • The variable A is a "matrix" implemented as a 1D array whose values are {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} then

I expect that:

  • P0 receives from A values {1,2,5,6}
  • P1 receives from A values {3,4,7,8}
  • P2 receives from A values {9,10,13,14}
  • P3 receives from A values {11,12,15,16}

as clearly shown here: strategy

2. My code

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <mpi.h>

bool is_divisible(int, int);
void build_mesh(MPI_Comm*, MPI_Comm*, MPI_Comm*, int, int, int, int, int*);
int *fill_matrix(int*, int, int);
int *fill_array(int*, int);
void print_matrix(int*, int, int, int, int);

int main(int argc, char *argv[])
{
    int process_rank, world_size;
    int mesh_rows, mesh_columns;
    int mesh_dimension = 2;
    int *process_coordinates;
    MPI_Comm mesh, mesh_r, mesh_c;
    int process_rank_mesh;
    int *A, *A_loc;
    int m, n; // input A square shape
    int mloc, nloc; // local A square shape

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &process_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &world_size);

    if (process_rank == 0) {
        m = n = 8; // multiple of world_size = 4
    }

    MPI_Bcast(&m, 1, MPI_INT, 0, MPI_COMM_WORLD);
    MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD);
    A = fill_matrix(A, m, n);

    if (process_rank == 0) 
        mesh_rows = 2;

    MPI_Bcast(&mesh_rows, 1, MPI_INT, 0, MPI_COMM_WORLD);

    if (is_divisible(world_size, mesh_rows))
        mesh_columns = world_size / mesh_rows;
    else {
        mesh_rows = 1;
        mesh_columns = world_size / mesh_rows;
    }
   
    MPI_Bcast(&mesh_rows, 1, MPI_INT, 0, MPI_COMM_WORLD);
    MPI_Bcast(&mesh_columns, 1, MPI_INT, 0, MPI_COMM_WORLD);

    process_coordinates = (int*) calloc(mesh_dimension, sizeof(int));
    build_mesh(&mesh, &mesh_r, &mesh_c, process_rank, world_size, mesh_rows, mesh_columns, process_coordinates);
    MPI_Comm_rank(mesh, &process_rank_mesh); 
 
    mloc = m / mesh_rows;
    nloc = m / mesh_columns;

    // Everything is correct up to now - The problem must be here: 
    // Distribution
    A_loc = (int*) calloc(mloc * nloc, sizeof(int));
    A_loc = (int*) calloc(mloc * nloc, sizeof(int));
    MPI_Datatype square_block;
    int stride = n;
    int count = mloc;
    int block_length = nloc;
    MPI_Type_vector(count, block_length, stride, MPI_INT, &square_block);
    MPI_Datatype square_block_resized;
    MPI_Type_create_resized(square_block, 0, sizeof(int), &square_block_resized);
    MPI_Type_commit(&square_block_resized);
    int *send_counts = (int*) calloc(world_size, sizeof(int));
    int *displs = (int*) calloc(world_size, sizeof(int));
    for (int i = 0; i < mesh_rows; i++) {
        for (int j = 0; j < mesh_columns; j++) {
            send_counts[i * mesh_columns + j] = 1;
            displs[i * mesh_columns + j] = i * n * block_length + j * block_length;
        }
    }
    MPI_Scatterv(A, send_counts, displs, square_block_resized, A_loc, 4, MPI_INT, 0, MPI_COMM_WORLD);

    // dbg distribution
    print_matrix(A, m, n, process_rank, -1); // original matrix
    for (int i = 0; i < world_size; i++) {
        MPI_Barrier(MPI_COMM_WORLD);
        print_matrix(A_loc, mloc, nloc, process_rank, i);
    }

    MPI_Finalize();
    return 0;
}

// 100% working - no need to check
bool is_divisible(int dividend, int divisor)
{
    return dividend % divisor == 0;
}

// 100% working - no need to check
void build_mesh(MPI_Comm *mesh, MPI_Comm *mesh_r, MPI_Comm *mesh_c, int process_rank, int world_size,
    int mesh_rows, int mesh_columns, int *process_coordinates) 
{
    int mesh_dimension = 2;
    int *mesh_n_dimension;
    int mesh_reorder = 0;
    int *mesh_period;
    int *remain_dims = (int*) calloc(mesh_dimension, sizeof(int));
    mesh_n_dimension = (int*) calloc(mesh_dimension, sizeof(int));
    mesh_n_dimension[0] = mesh_rows;
    mesh_n_dimension[1] = mesh_columns;
    mesh_period = (int*) calloc(mesh_dimension, sizeof(int));
    mesh_period[0] = mesh_period[1] = 0;
    MPI_Cart_create(MPI_COMM_WORLD, mesh_dimension, mesh_n_dimension, mesh_period, mesh_reorder, mesh);
    MPI_Cart_coords(*mesh, process_rank, mesh_dimension, process_coordinates);
    remain_dims[0] = 0;  
    remain_dims[1] = 1;
    MPI_Cart_sub(*mesh, remain_dims, mesh_r);
    remain_dims[0] = 1;
    remain_dims[1] = 0;
    MPI_Cart_sub(*mesh, remain_dims, mesh_c);
}

// 100% working - no need to check
int *fill_matrix(int *A, int m, int n)
{
    int k = 0;
    A = (int*) calloc(m * n, sizeof(int));
    for (int i = 0; i < m; i++) {
        for (int j = 0; j < n; j++) {
            A[i * n + j] = ++k;
        }
    }
    return A;
}

// 100% working - no need to check
void print_matrix(int *A, int m, int n, int process_rank, int id)
{
    if (id == -1) {
        if (process_rank == 0) {
            printf("Original matrix:\n");
            for (int i = 0; i < m; i++) {
                for (int j = 0; j < n; j++) {
                    printf("%d\t", A[i * n + j]);
                }
                printf("\n");
            }
            printf("\n");
        }
    }
    if (process_rank == id) {
        printf("P%d local matrix:\n", id);
        for (int i = 0; i < m; i++) {
            for (int j = 0; j < n; j++) {
                printf("%d\t", A[i * n + j]);
            }
            printf("\n");
        }
        printf("\n");
    }
}

3. Results

This works fine if m = 4:

Original matrix:
1       2       3       4
5       6       7       8
9       10      11      12
13      14      15      16

P0 local matrix:
1       2
5       6

P1 local matrix:
3       4
7       8

P2 local matrix:
9       10
13      14

P3 local matrix:
11      12
15      16

but unfortunately I am getting stuck if m != 4 but it's a multiple of world_size as you can see here (e.g. m = 8, 12...):

[Air:01914] *** An error occurred in MPI_Scatterv
[Air:01914] *** reported by process [138870785,2]
[Air:01914] *** on communicator MPI_COMM_WORLD
[Air:01914] *** MPI_ERR_TRUNCATE: message truncated
[Air:01914] *** MPI_ERRORS_ARE_FATAL (processes in this communicator will now abort,
[Air:01914] ***    and potentially your MPI job)
[Air:01911] PMIX ERROR: UNREACHABLE in file server/pmix_server.c at line 2198
[Air:01911] PMIX ERROR: UNREACHABLE in file server/pmix_server.c at line 2198
[Air:01911] 2 more processes have sent help message help-mpi-errors.txt / mpi_errors_are_fatal
[Air:01911] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
  • Can you help me? displs[] values are correct...
Solution

  • I simply solved replacing

    MPI_Scatterv(A, send_counts, displs, square_block_resized, A_loc, 4, MPI_INT, 0, MPI_COMM_WORLD);

    with

    MPI_Scatterv(A, send_counts, displs, square_block_resized, A_loc, mloc * nloc, MPI_INT, 0, MPI_COMM_WORLD);

    that is replacing the hard-coded 4 with mloc * nloc