Sending from all processor to root, using MPI_Send and MPI_Recv.

Question

I have trouble to collect some data from all processors to the root, here is one example of what I want to do:

I have a couple of pairs (actually they are edges) in each processor and ideally want to send them to the root, or if there is no way I can send their corresponding index (one number instead of pairs.

For example:

Processor 0: sends {(0,5), (1,6)} to root, or it sould send {5,17}
Processor 1: sends {(2,3)} to root, or it sould send {14}
Processor 2: sends {} to root, or it sould send {}
Processor 3: sends {(4,0)} to root, or it sould send {20}

I am wondering what is the best way to store the pairs or numbers and send and receive them. I ideally I prefer to store them in a 2d vector since from the beginning I don`t know how much space I need, and receive them again in a 2D vector. I know it might not possible or might be very complicated.

This is a pseudocode of the procedure I am looking for but don`t know how to implement in MPI.

vector<vector<int > >allSelectedEdges;
vector<vector<int > >selectedEdgesLocal; 
int edgeCount=0;   
if(my_rank!=0){                          
        for(int i = 0; i < rows; ++i)
            for(int j = 0; j < nVertex; ++j)
                if (some conditions)
                {
                    vector<int> tempEdge;
                    tempEdge.push_back(displs[my_rank]+i);
                    tempEdge.push_back(j);
                    selectedEdgesLocal.push_back(tempEdge);
                    edgeCount++;
                }
        }
        "send selectedEdgesLocal to root"
}else
{
      "root recieve sselectedEdgesLocal and store in allSelectedEdges"
}

I thought about MPI_Gatherv as well but seems that doesn`t help. Got the idea from here

vector<vector<int > >selectedEdgesLocal; 
int edgeCount=0;                            
for(int i = 0; i < rows; ++i)
    for(int j = 0; j < nVertex; ++j)
        if (some conditions)
        {
            vector<int> tempEdge;
            tempEdge.push_back(displs[my_rank]+i);
            tempEdge.push_back(j);
            selectedEdgesLocal.push_back(tempEdge);
            edgeCount++;
         }
int NumdgesToAdd;
MPI_Reduce(&edgeCount, &NumdgesToAdd, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);

vector<vector<int > > allSelectedEdges(NumdgesToAdd);

int rcounts[comm_size];
int rdisp[comm_size];
int sumE=0;
for(int i=0; i<comm_size; ++i) {
    rcounts[i] = edgeCount;
    rdisp[i]=sumE;
    sumE+=edgeCount;
}

MPI_Gatherv(&selectedEdgesLocal.front(), rcounts[my_rank], MPI_INT, &allSelectedEdges.front(), rcounts, rdisp, MPI_INT, 0, MPI_COMM_WORLD);

Answer 1

I update my code as follows, and it is working now.

vector <int> selectedEdgesIndicesLocal;

    int edgeCount=0;
    for(int i = 0; i < rows; ++i)
        for(int j = 0; j < nVertex; ++j)
            if (some condistions)
            {
                int index=...;
                selectedEdgesIndicesLocal.push_back(index);
                edgeCount++;
            }

    int NumEdgesToAdd;
    MPI_Reduce(&edgeCount, &NumEdgesToAdd, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);

    int *edgeCountsInRoot;
    if (my_rank == 0)edgeCountsInRoot = (int *)malloc(comm_size * sizeof(int));
    MPI_Gather(&edgeCount, 1, MPI_INT, edgeCountsInRoot, 1, MPI_INT, 0,MPI_COMM_WORLD);

    int *allSelectedIndicesEdges;
    if (my_rank == 0)allSelectedIndicesEdges = (int *)malloc(NumEdgesToAdd * sizeof(int));

    int * edgeCounts, *edgeDisp;

    cout<<edgeCount<<endl;
    if (my_rank==0) {
        edgeCounts= (int *)malloc(comm_size * sizeof(int));
        edgeDisp= (int *)malloc(comm_size * sizeof(int));
        int edgeSum=0;
        for(int i=0; i<comm_size; ++i) {
            edgeCounts[i] = edgeCountsInRoot[i];
            edgeDisp[i]=edgeSum;
            edgeSum+=edgeCountsInRoot[i];
        }
    }
    MPI_Gatherv(&selectedEdgesIndicesLocal.front(), edgeCount, MPI_INT, &allSelectedIndicesEdges[0], edgeCounts,edgeDisp, MPI_INT, 0, MPI_COMM_WORLD);