Sync_Time_MPI.c

Go to the documentation of this file.

/* ############################ MALP License ############################## */
/* # Fri Jan 18 14:00:00 CET 2013                                         # */
/* # Copyright or (C) or Copr. Commissariat a l'Energie Atomique          # */
/* #                                                                      # */
/* # This software is governed by the CeCILL-C license under French law   # */
/* # and abiding by the rules of distribution of free software.  You can  # */
/* # use, modify and/ or redistribute the software under the terms of     # */
/* # the CeCILL-C license as circulated by CEA, CNRS and INRIA at the     # */
/* # following URL http://www.cecill.info.                                # */
/* #                                                                      # */
/* # The fact that you are presently reading this means that you have     # */
/* # had knowledge of the CeCILL-C license and that you accept its        # */
/* # terms.                                                               # */
/* #                                                                      # */
/* # Authors:                                                             # */
/* #   - BESNARD Jean-Baptiste jean-baptiste.besnard@cea.fr               # */
/* #                                                                      # */
/* ######################################################################## */
#include "Sync_Time_MPI.h"
#include "Trace_Comm.h"

void compute_sync_tree( struct sync_tree_conf *conf,
                        void (*action)(struct sync_tree_conf *next_conf, void *arg),
                        void (*per_child_act)(struct sync_tree_conf *conf, void *arg ),
                        void (*post_action)(struct sync_tree_conf *next_conf, void *arg),
                        void *arg
                      )
{
        if (conf->comm_size <= 0) {
                return;
        }
        
        /* Do node local action */
        if( action && conf->source_node != 0)
                (action)(conf, arg);
        

        int next_pow = conf->current_pow;
        /* If there is no child just call post action */
        while( conf->comm_size < conf->target_node + next_pow  ) {
                next_pow >>= 1;

                if( !next_pow ) {
                        if( post_action )
                                (post_action)( conf, arg );
                        return;
                }
        }



        /* Compute tree then call per child action */
        uint32_t chld_count = 0;
        struct sync_tree_conf next_conf[64];

        while( next_pow ) {

                next_conf[chld_count].comm_size = conf->comm_size;
                next_conf[chld_count].source_node = conf->target_node ;
                next_conf[chld_count].target_node = conf->target_node + next_pow ;
                next_conf[chld_count].current_pow = next_pow >> 1;
                next_conf[chld_count].offset = *((long long int *)arg);

                PMPI_Send( &next_conf[chld_count], sizeof(struct sync_tree_conf), MPI_CHAR, next_conf[chld_count].target_node - 1, 123, Trace_Comm_get());

                chld_count++;

                if( 64 < chld_count) {
                        printf("WARNING NOT ENOUGH CONF CELL PLEASE AUGMENT IN FILE %s @ %d\n", __FILE__, __LINE__);
                        abort();
                }

                next_pow >>= 1;
        }

        if( chld_count ) {
                do {
                        if( per_child_act )
                                (per_child_act)(&next_conf[chld_count - 1], arg);

                        chld_count--;
                } while( chld_count );

        }

        /* Call post action */
        if( post_action )
                (post_action)( conf, arg );

}


void bootstrap_sync_tree( void (*action)(struct sync_tree_conf *next_conf, void *arg),
                          void (*per_child_act)(struct sync_tree_conf *conf , void *arg),
                          void (*post_action)(struct sync_tree_conf *next_conf, void *arg),
                          void *arg
                        )
{

        int rank = 0;
        int comm_size = 0;

        PMPI_Comm_rank( Trace_Comm_get(), &rank );
        PMPI_Comm_size( Trace_Comm_get(), &comm_size );

        struct sync_tree_conf conf;

        if( rank == 0 ) {


                conf.comm_size = comm_size;
                conf.source_node = 0;
                conf.offset = 0;
                conf.target_node = 1;
                conf.current_pow = nearest_pow (comm_size);

                compute_sync_tree( &conf,  action, per_child_act, post_action, arg);


        } else {

                MPI_Status st;
                PMPI_Recv( &conf, sizeof( struct sync_tree_conf), MPI_CHAR, MPI_ANY_SOURCE, 123, Trace_Comm_get(), &st );

                compute_sync_tree( &conf,  action, per_child_act, post_action, arg);
        }

}



void sync_server( int dest_rank )
{
        //printf("ENTERING SERVER FOR CHILD %d\n", dest_rank);

        uint64_t T0 = 0, T1 = 0, Tr = 0;
        long long int DTr = 1;
        int try_count = 0;

        long long int sum = 0 , count = 0;
        int i = 0;

        long long int round_trip = 0;
        MPI_Status st;

        while( try_count < MAX_LOOP_TRY ) {
                DTr = 1;

                sum = 0;
                count = 0;
                round_trip = 0;

                for( i = 0 ; i < MAX_AVG ; i++ ) {
                        T0 = Timer_tsc();

                        PMPI_Send(&DTr, 1, MPI_LONG_LONG_INT, dest_rank, SYNC_TAG, Trace_Comm_get());

                        PMPI_Recv(&Tr, sizeof(uint64_t), MPI_CHAR, dest_rank, SYNC_TAG, Trace_Comm_get(), &st );

                        T1 = Timer_tsc();

                        round_trip += T1 - T0;
                        sum += ((T0 + T1) / 2) - Tr;
                        count++;
                }

                DTr = sum  / count;
                round_trip = round_trip / count;


                T0 = Timer_tsc();


                if( DTr == 0 )
                        DTr = 1; //avoid sending the cancel state

                PMPI_Send(&DTr, 1, MPI_LONG_LONG_INT, dest_rank, SYNC_TAG, Trace_Comm_get());

                PMPI_Recv(&Tr, 8, MPI_CHAR, dest_rank, SYNC_TAG, Trace_Comm_get(), &st );

                T1 = Timer_tsc();


                if( abs_diff( ((T0 + T1)/2) , Tr ) < (round_trip / 500) ) {
                        //printf("SYNCED %d DELTA = %ld TRIES = %d\n",dest_rank,  abs_diff( ((T0 + T1)/2) , Tr ), try_count);

                        //sending cancel state
                        DTr = 0;
                        PMPI_Send(&DTr, 1, MPI_LONG_LONG_INT, dest_rank, SYNC_TAG, Trace_Comm_get());
                        //--------------------
                        return ;
                }

                try_count++;
        }

        //sending cancel state
        DTr = 0;
        PMPI_Send(&DTr, 1, MPI_LONG_LONG_INT, dest_rank, SYNC_TAG, Trace_Comm_get());
        //-------------------

}

void sync_client(int parent, void *poffset)
{
        //printf("ENTERING CLIENT FOR PARENT %d\n", parent);

        long long int DTr = 0;
        uint64_t Tr = 0;
        long long int sync_offset = 0;
        MPI_Status st;
        int counter = 0;

        do {

                PMPI_Recv(&DTr, 1, MPI_LONG_LONG_INT, parent, SYNC_TAG, Trace_Comm_get(), &st );

                if( DTr )
                        sync_offset = DTr;
                Tr = Timer_tsc() + sync_offset;

                if( DTr )
                        PMPI_Send(&Tr, 8, MPI_CHAR, parent, SYNC_TAG, Trace_Comm_get());


                counter++;


        } while( DTr != 0 );

        //printf("SLAVE Getting offset %ld from %d\n", sync_offset, parent);

        long long int *offset = (long long int *)poffset;
        *offset = sync_offset;
}


void perchild_host(struct sync_tree_conf *c, void *arg)
{
        int rank = 0;
        PMPI_Comm_rank( Trace_Comm_get(), &rank );
        //printf("PER CHILD [%d] %d -> %d (%d)\n", rank, c->source_node - 1, c->target_node - 1, c->current_pow);

        sync_server( c->target_node - 1 );

}


void post_client(struct sync_tree_conf *c, void *arg)
{
        int rank = 0;
        PMPI_Comm_rank( Trace_Comm_get(), &rank );
        //printf("POST ACTION [%d] %d -> %d (%d)\n", rank, c->source_node - 1, c->target_node - 1, c->current_pow);
        if(c->source_node != 0 )
                sync_client(c->source_node - 1, arg);
}

void action_set_off(struct sync_tree_conf *c, void *poffset)
{
        long long int *offset = (long long int *)poffset;
        int rank = 0;
        PMPI_Comm_rank( Trace_Comm_get(), &rank );


        *offset += c->offset;

        //printf("Rank %d has time offset : %lld \n", rank,  *offset );

}


void action_val(struct sync_tree_conf *c, void *arg)
{
        int rank = 0;
        PMPI_Comm_rank( Trace_Comm_get(), &rank );
        printf("SYNCED %d <-> %d\n", /*rank,*/ c->source_node - 1, c->target_node - 1  /*,c->current_pow*/);
}

void sync_mpi()
{

        int rank = 0;
        PMPI_Comm_rank( Trace_Comm_get(), &rank );

        if( !Process_time_origin )
                Timer_set_origin();

        //#warning can be commented out for cleaner output ! (action_val)
        bootstrap_sync_tree( /*action_val*/ NULL , perchild_host, post_client, &Process_Sync_Offset);
        bootstrap_sync_tree( action_set_off, NULL, NULL, &Process_Sync_Offset );
}