Measuring data transfer between processes in OpenMPI

Question

I am working in a cluster with ubuntu and OpenMPI 1.4. My code is working good but i would like to measure the time that take send data between root process and slave nodes.

I think my code doesn't give me the correct information.

void invertColor_Parallel(struct image *im, int size, int rank,clock_t *sendTime)
{
     clock_t begin;

     int i,j,lum,aux,r;

     int total_pixels = (*im).ih.width * (*im).ih.height;
     int qty = total_pixels/(size-1);
     int rest = total_pixels % (size-1);
     MPI_Status status;

     //printf("\n%d\n", rank);

     if(rank == 0)
     {
        begin = MPI_Wtime();
         for(i=1; i<size; i++){
         j = i*qty - qty;
         aux = j;

         if(rest != 0 && i==size-1) {qty=qty+rest;} //para distrubuir toda la carga
         //printf("\nj: %d  qty: %d  rest: %d\n", j, qty, rest);


         MPI_Send(&aux, 1, MPI_INT, i, MASTER_TO_SLAVE_TAG+1, MPI_COMM_WORLD);
         MPI_Send(&qty, 1, MPI_INT, i, MASTER_TO_SLAVE_TAG+2, MPI_COMM_WORLD);
         MPI_Send(&(*im).array[j], qty*3, MPI_BYTE, i, MASTER_TO_SLAVE_TAG, MPI_COMM_WORLD);


         //printf("\nSending to node=%d, sender node=%d\n", i, rank);
        }
        *sendTime+=MPI_Wtime()-begin;

     }
     else
     {
        MPI_Recv(&aux, 1, MPI_INT, MPI_ANY_SOURCE, MASTER_TO_SLAVE_TAG+1, MPI_COMM_WORLD,&status);
        MPI_Recv(&qty, 1, MPI_INT, MPI_ANY_SOURCE, MASTER_TO_SLAVE_TAG+2, MPI_COMM_WORLD,&status);

        pixel *arreglo = (pixel *)calloc(qty, sizeof(pixel));
        MPI_Recv(&arreglo[0], qty*3, MPI_BYTE, MPI_ANY_SOURCE, MASTER_TO_SLAVE_TAG, MPI_COMM_WORLD,&status);
        //printf("Receiving node=%d, message=%d\n", rank, aux);

        for(i=0;i<qty;i++)
        {
            arreglo[i].R = 255-arreglo[i].R;
            arreglo[i].G = 255-arreglo[i].G;
            arreglo[i].B = 255-arreglo[i].B;
        }

        MPI_Send(&aux, 1, MPI_INT, 0, SLAVE_TO_MASTER_TAG+1, MPI_COMM_WORLD);
        MPI_Send(&qty, 1, MPI_INT, 0, SLAVE_TO_MASTER_TAG+2, MPI_COMM_WORLD);
        MPI_Send(&arreglo[0], qty*3, MPI_BYTE, 0, SLAVE_TO_MASTER_TAG, MPI_COMM_WORLD);

        free(arreglo);
     }


    if (rank==0){
    //printf("\nrank: %d\n", rank);
        begin=MPI_Wtime();
        for (i=1; i<size; i++) // untill all slaves have handed back the processed data
        {
            MPI_Recv(&aux, 1, MPI_INT, MPI_ANY_SOURCE, SLAVE_TO_MASTER_TAG+1, MPI_COMM_WORLD, &status);
            MPI_Recv(&qty, 1, MPI_INT, status.MPI_SOURCE, SLAVE_TO_MASTER_TAG+2, MPI_COMM_WORLD, &status);
            MPI_Recv(&(*im).array[aux], qty*3, MPI_BYTE, status.MPI_SOURCE, SLAVE_TO_MASTER_TAG, MPI_COMM_WORLD, &status);
        }
        *sendTime+=MPI_Wtime()-begin;
    }
}

void runningTime(clock_t begin, clock_t end,clock_t sendTime)
{
    double time_spent;

    time_spent = (double)(end - begin) / CLOCKS_PER_SEC;
    printf("Tiempo con Latencia: %f\n", time_spent);
    time_spent-=(double) sendTime/CLOCKS_PER_SEC;
    printf("Tiempo de envio: %f\n", time_spent);
    printf("Tiempo de latencia: %f\n\n", (double) sendTime/CLOCKS_PER_SEC);
}

int main(int argc, char *argv[])
{
    //////////time counter
    clock_t begin,sendTime=(clock_t) 0.0;


    int rank, size;

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Status status;

    int op = (int)atof(argv[1]);
    struct image image2;

    if (rank==0)
    {
        loadImage(&image2, argv[2]);
    }

    //Broadcast the user's choice to all other ranks
    MPI_Bcast(&op, 1, MPI_INT, 0, MPI_COMM_WORLD);


    switch(op)
    {
        case 1:
                if (rank==0) {begin = clock();}
                MPI_Barrier(MPI_COMM_WORLD);
                invertColor_Parallel(&image2, size, rank,&sendTime);
                MPI_Barrier(MPI_COMM_WORLD);
                if (rank==0) {runningTime(begin, clock(),sendTime); printf("Se invirtieron los colores de la imagen\n\n");}
                break;
    }

    MPI_Barrier(MPI_COMM_WORLD);

    if (rank==0)
    {
        saveImage(&image2, argv[3]);
        free(image2.array);
    }

    MPI_Finalize();

    return 0;
}

I want to measure the data transfer time between the processes and the entire time of the function ivertColor_Parallel. What am i doing wrong?.

Thank you very much.

Answer 1

MPI_Wtime() returns a double precision floating-point number that gives the amount of seconds passed since some point in the past. clock_t is an integer type. First, you lose the fractional (sub-second) precision when you convert the result from MPI_Wtime() to clock_t. Second, after you divide the resultant clock_t value by CLOCKS_PER_SEC (equal to 1000000 on POSIX systems like Linux) the result is always going to be 0 unless more than 11 days and 14 hours have passed between the two calls to MPI_Wtime().

MPI_Wtime() usage is as simple as:

double begin = MPI_Wtime();
...
double end = MPI_Wtime();
printf("Time elapsed: %f seconds\n", end - begin);