VMPI.c

Go to the documentation of this file.

/* ############################ MALP License ############################## */
/* # Fri Jan 18 14:00:00 CET 2013                                         # */
/* # Copyright or (C) or Copr. Commissariat a l'Energie Atomique          # */
/* #                                                                      # */
/* # This software is governed by the CeCILL-C license under French law   # */
/* # and abiding by the rules of distribution of free software.  You can  # */
/* # use, modify and/ or redistribute the software under the terms of     # */
/* # the CeCILL-C license as circulated by CEA, CNRS and INRIA at the     # */
/* # following URL http://www.cecill.info.                                # */
/* #                                                                      # */
/* # The fact that you are presently reading this means that you have     # */
/* # had knowledge of the CeCILL-C license and that you accept its        # */
/* # terms.                                                               # */
/* #                                                                      # */
/* # Authors:                                                             # */
/* #   - BESNARD Jean-Baptiste jean-baptiste.besnard@cea.fr               # */
/* #                                                                      # */
/* ######################################################################## */
#include "VMPI.h"

#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <CRC64.h>
#include <stdint.h>
#include <errno.h>
#include "MALP_Config.h"

struct VMPI_Status __vmpi_status = {0};

int VMPI_get_new_tag()
{
       int ret = ++__vmpi_status.current_tag;

      if( ret <  VMPI_NG_MAXIMUM_TAGS )
       {
               return ret;
       }
       else
       {
               printf("Maximum number of booked tags exhausted\n");
               abort();
       }

       return VMPI_ERROR;
}


int VMPI_Enabled()
{
        return __vmpi_status.vmpi_enabled;
}


void VMPI_Display_desc(VMPI_Partition_desc *desc)
{
        if( !desc )
                return;

        printf("===============\n" );
        printf("Id : %d\n", desc->id );
        printf("Size : %d\n", desc->size );
        printf("Root : %d\n", desc->root_id );
        printf("Name : %s\n", desc->name );
        printf("===============\n" );
}

void VMPI_Display_descs()
{
        int i = 0;
        
        for( i = 0 ; i < VMPI_Get_partition_count() ; i++ )
        {
                VMPI_Display_desc(&__vmpi_status.partition_descs[i]);
        }
}


void VMPI_Print_map( VMPI_Map *map )
{
        int rank;
        PMPI_Comm_rank( MPI_COMM_WORLD, &rank );
        
        int i;
        
        printf("======================\n");
        for( i=0 ; i <  map->count; i++ )
                printf("%d --> %d\n", rank, map->ranks[i] );
        
        printf("======================\n");
}

int VMPI_Map_partitions( int target ,  VMPI_Map_mode mode, VMPI_Map *map )
{
        int count, *ranks;
        VMPI_Partition_desc *pa, *pb, *local_p, *remote_p;
        VMPI_Partition_desc *tx_part, *rx_part;
        int partition_rank;
        int global_rank;
        int target_rank, source_rank;
        int i,j;
        int incoming_count;
        int *incoming;
        int source_target;
        MPI_Status s;
        VMPI_Map_mode remote_mode;

        pa = VMPI_Get_desc();
        pb = VMPI_Get_desc_by_id(target);

        /* Does these partitions exist ? */
        if( !pa || !pb )
        {
                return VMPI_ERROR;
        }

        /* Store partitions in term of local and remote */
        if( pa->id == VMPI_Get_partition_id() )
        {
                local_p = pa;
                remote_p = pb;
        }
        else
        {
                local_p = pb;
                remote_p = pa;
        }
        
        if( PMPI_Comm_rank( VMPI_Get_partition_comm(), &partition_rank ) != MPI_SUCCESS )
        {
                return VMPI_ERROR;
        }

        if( PMPI_Comm_rank( VMPI_Get_vmpi_comm(), &global_rank ) != MPI_SUCCESS )
        {
                return VMPI_ERROR;
        }

        /* Each partition root exchanges its mode to check their equality */
        if( partition_rank == 0 )
        {
                if( PMPI_Sendrecv((void *)&mode, sizeof(VMPI_Map_mode), MPI_CHAR, remote_p->root_id, VMPI_NG_ARBITRARY_VALUE + 42,
                                                  (void *)&remote_mode, sizeof(VMPI_Map_mode), MPI_CHAR, remote_p->root_id, VMPI_NG_ARBITRARY_VALUE + 42,
                                                  MPI_COMM_WORLD, &s) != MPI_SUCCESS )
                {
                        return VMPI_ERROR;
                }

                /* Check for equality */
                if( remote_mode != mode )
                {
                        printf("Mismatching modes between partition %s and %d\n",  VMPI_Get_desc()->name, target);
                        abort();
                }
        }

        /* Run checks for map fixed */
        if( (pa->size != pb->size) && ( mode == VMPI_MAP_FIXED ) )
        {
                printf("(%s[%d] -> %d[%d] ) : VMPI_MAP_FIXED is not possible on communicators of different sizes\n",VMPI_Get_desc()->name, pa->size, target, pb->size);
                abort();
        }

        /* The larger comm is the sender 
         * In case of equality we use the id*/
        
        if(pa->size != pb->size)
        {
                if( pa->size < pb->size )
                {
                        tx_part=pb;
                        rx_part=pa;
                }
                else
                {
                        tx_part=pa;
                        rx_part=pb;
                }
        }
        else
        {
                if( pa->id < pb->id )
                {
                        tx_part=pb;
                        rx_part=pa;
                }
                else
                {
                        tx_part=pa;
                        rx_part=pb;
                }
        }

        /* Each rank of the larger comm sends its ID 
         * While each rank of the lower comm sends back its own
         * being dispatched by the root rank*/
        target_rank = 0;
        incoming_count = 0;
        incoming = NULL;

        /* As the larger comm is the sender only Rx can get multiple incoming */

        if( tx_part->id == VMPI_Get_partition_id() )
        {

                /* Sender */
                /* Send Id to Rx comm */
                if( PMPI_Send((void *)&global_rank, 1, MPI_INT, rx_part->root_id, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                return VMPI_ERROR;

                
                incoming_count = 1;
                ranks = malloc( sizeof( int ) );
                
                if( !ranks )
                {
                        return VMPI_MEM;
                }
                *ranks = -1;
                count = 1;
                

                /* Receive matching ID */
                if( PMPI_Recv((void *)ranks, 1, MPI_INT, rx_part->root_id, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm(), &s) != MPI_SUCCESS )
                {
                        return VMPI_ERROR;
                }
        }
        else
        {
                /* Receiver */
                /* If we are the root of rx partition */

                if( global_rank == rx_part->root_id )
                {
                        for( i = 0 ; i < tx_part->size ; i++ )
                        {
                                if( PMPI_Recv((void *)&source_rank, 1, MPI_INT, tx_part->root_id + i, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm(), &s) != MPI_SUCCESS )
                                        return VMPI_ERROR;
                                
                                switch( mode )
                                {
                                        case VMPI_MAP_FIXED:
                                        case VMPI_MAP_ROUND_ROBIN :
                                                target_rank = i % rx_part->size;
                                                break;
                                        case VMPI_MAP_RANDOM :
                                                target_rank = rand() % rx_part->size;
                                                break;
                                        default :
                                                printf("No such mode in %s\n", __FUNCTION__ );
                                                abort();
                                }
                                
                                /* This rank wants the root as target */
                                if( target_rank == 0 )
                                {
                                        incoming_count++;
                                        count = incoming_count;
                                        incoming = realloc( incoming, incoming_count * sizeof( int ) );
                                        
                                        if( !incoming )
                                        {
                                                return VMPI_MEM;
                                        }

                                        incoming[incoming_count - 1] = source_rank;


                                        if( PMPI_Send((void *)&global_rank, 1, MPI_INT, source_rank, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                        {
                                                return VMPI_ERROR;
                                        }
                                }
                                else
                                {
                                        /* Relay to the target process */
                                        if( PMPI_Send((void *)&source_rank, 1, MPI_INT, rx_part->root_id + target_rank,
                                                                        VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                        {
                                                return VMPI_ERROR;
                                        }
                                        
                                        /* Send target to source */
                                        source_target = rx_part->root_id + target_rank;
                                        if( PMPI_Send((void *)&source_target, 1, MPI_INT, source_rank, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                        {
                                                return VMPI_ERROR;
                                        }
                                }

                        }
                        
                        /* Once we are done Broadcast the end to all slaves in the rx_set */
                        for( i = 1 ; i < rx_part->size ; i++ )
                        {
                                source_rank = -1;
                                if( PMPI_Send((void *)&source_rank, 1, MPI_INT, rx_part->root_id + i, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                {
                                        return VMPI_ERROR;
                                }
                        }
                        
                        
                }
                else
                {
                        while(1)
                        {
                                if( PMPI_Recv((void *)&source_rank, 1, MPI_INT, rx_part->root_id, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm(), &s) != MPI_SUCCESS )
                                        return VMPI_ERROR;

                                if( source_rank < 0 )
                                        break;

                                incoming_count++;
                                count = incoming_count;
                                incoming = realloc( incoming, incoming_count * sizeof( int ) );
                                        
                                if( !incoming )
                                {
                                        return VMPI_ERROR;
                                }

                                incoming[incoming_count - 1] = source_rank;
                        }
                }

                ranks = incoming;
        }

        /*Concatenate to map */
        int prev_count = map->count;
        map->count += count;
        
        if( prev_count )
        {
                map->ranks = realloc( map->ranks, sizeof(int) * map->count );
                
                if( !map->ranks )
                {
                        perror("realloc");
                        exit(1);
                }
                
                memcpy( map->ranks + prev_count , ranks, count * sizeof( int ) );
                
                free( ranks );
        }
        else
        {
                map->ranks = ranks;
        }

        /* Sort ranks */
        do
        {
                j = 0;
                for( i = 0 ; i < map->count - 2 ; i++ )
                {
                        if( map->ranks[i+1] < map->ranks[i] )
                        {
                                target_rank = map->ranks[i];
                                map->ranks[i] = map->ranks[i+1];
                                map->ranks[i+1] = target_rank;
                                j = 1;
                        }
                        
                }
                
        }while(j != 0 );

        return VMPI_SUCCESS;
}


void VMPI_Fill_desc_command(VMPI_Partition_desc *desc, int argc, char **argv )
{
        int len = 0;
        int i = 0;
        
        /* Fill in program name */
        memset( desc->program_name, 0, VMPI_PNAME_LEN);
        /* Fill in command line */
        memset( desc->command_line, 0, VMPI_COMMAND_LEN);

        if( argv )
        {
                /* Fill in program name */
                snprintf(desc->program_name, VMPI_PNAME_LEN, "%s", argv[0] );
                
                /* Fill in command line */
                while( i < argc )
                {
                        len += strlen( argv[i] ) + 1;

                        if(VMPI_COMMAND_LEN <= len )
                                break;
        
                        strcat(desc->command_line, argv[i]);

                        if( i != (argc - 1) )
                                strcat(desc->command_line, " ");

                        i++;
                }
        }
}



int VMPI_Dispatch_descs(MPI_Comm partition_comm, int argc, char **argv)
{
        int partition_rank, global_rank, i;
        MPI_Status s;

        PMPI_Comm_rank( partition_comm, &partition_rank);
        PMPI_Comm_rank( VMPI_Get_vmpi_comm(), &global_rank);

        __vmpi_status.partition_descs = malloc( sizeof( VMPI_Partition_desc ) * VMPI_Get_partition_count() );
        
        if( !__vmpi_status.partition_descs )
                return VMPI_ERROR;
        
        if( global_rank == 0 )
        {
                /* 0 is a root */
                PMPI_Comm_size(partition_comm,&__vmpi_status.partition_descs[0].size);
                __vmpi_status.partition_descs[0].id = VMPI_Get_partition_id();
                __vmpi_status.partition_descs[0].root_id = 0;
                sprintf(__vmpi_status.partition_descs[0].name, "%s", __vmpi_status.partition_name);
                VMPI_Fill_desc_command(&__vmpi_status.partition_descs[0], argc, argv );

                if( 1 < VMPI_Get_partition_count() )
                {
                        /*receive datas from other roots */
                        for( i = 1 ; i < VMPI_Get_partition_count(); i++ )
                        {
                                /* Receive descs */
                                if( PMPI_Recv((void *)&__vmpi_status.partition_descs[i], sizeof( VMPI_Partition_desc),
                                                           MPI_CHAR, MPI_ANY_SOURCE, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm(), &s) != MPI_SUCCESS )
                                        return VMPI_ERROR;

                        }
                }

                /* Sort by ID as we have no clue of the receive order */
                VMPI_Partition_desc tmp;
                VMPI_Partition_desc *desc;
                
                int did_swap = 0;
                
                do
                {
                        did_swap = 0;

                        for( i = 0; i < VMPI_Get_partition_count(); i++ )
                        {
                                desc = &__vmpi_status.partition_descs[i];

                                if( desc->id != i )
                                {
                                        tmp = __vmpi_status.partition_descs[desc->id];
                                        __vmpi_status.partition_descs[desc->id] = __vmpi_status.partition_descs[i];
                                        __vmpi_status.partition_descs[i] = tmp;
                                        did_swap = 1;
                                }
                        }
                }while( did_swap );


        }
        else
        {
                
                if( partition_rank == 0 )
                {
                        /* Fill in local partition desc */
                        PMPI_Comm_size(partition_comm,&__vmpi_status.partition_descs[0].size);
                        __vmpi_status.partition_descs[0].id = VMPI_Get_partition_id();
                        __vmpi_status.partition_descs[0].root_id = global_rank;
                        sprintf(__vmpi_status.partition_descs[0].name, "%s", __vmpi_status.partition_name);
                        VMPI_Fill_desc_command(&__vmpi_status.partition_descs[0], argc, argv );
                        
                                        
                        if( PMPI_Send((void *)__vmpi_status.partition_descs, sizeof( VMPI_Partition_desc), MPI_CHAR, 0, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                        return VMPI_ERROR;
                }
        }



        /* Now that 0 is aware of everything broadcast it */
        if( PMPI_Bcast( (void *)__vmpi_status.partition_descs, VMPI_Get_partition_count() * sizeof( VMPI_Partition_desc), MPI_CHAR, 0, VMPI_Get_vmpi_comm() ) != MPI_SUCCESS )
                return VMPI_ERROR;

        return VMPI_SUCCESS;
}

uint64_t VMPI_Compute_job_crc( int argc, char **argv )
{
        char *job_string;
        int job_string_len, i;
        uint64_t ret;

        /*Compute jobstring len */
        job_string_len = 0;
        for( i = 0 ; i < argc ; i++ )
                job_string_len += strlen( argv[i] );
        
        job_string_len++;
        job_string = malloc( job_string_len );
        
        if( !job_string )
                return 0;

        /* Concatenate command and args */
        *job_string = '\0';
        
        for( i = 0 ; i < argc ; i++ )
                strcat( job_string, argv[i] );
        
        /* Hash */
        ret = MALP_Trace_crc64(job_string, job_string_len);
        
        free( job_string );
        
        return ret;
}

struct TMP_Partition_desc
{
        uint64_t crc;   
        int id;         
};

int VMPI_Detect_Partitions(uint64_t job_crc, int *partition_count)
{
        /* We use a ring to avoid the possibly large array of non scalable MPI_Gather 
         * Even though MPI_Comm_split will do it ...*/
        
        struct TMP_Partition_desc local_desc;
        struct TMP_Partition_desc *remote_desc;
        MPI_Status s;
        int number_of_desc;
        int i, found;
        int global_size, global_rank;

        PMPI_Comm_rank( VMPI_Get_vmpi_comm(), &global_rank);
        PMPI_Comm_size( VMPI_Get_vmpi_comm(), &global_size);

        /* Fill in the local desc */
        local_desc.crc = job_crc;
        local_desc.id = 0;

        number_of_desc = 1;

        if( global_rank == 0 )
        {
                
                /* Send it to the next process */
                if( 1 < global_size )
                {
                        if( PMPI_Send((void *)&local_desc, sizeof( struct TMP_Partition_desc), MPI_CHAR, 1, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                return VMPI_ERROR;
                }

        }
        else
        {
                /* Probe to guess the number of incoming descs */
                if( PMPI_Probe(global_rank - 1, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm(), &s) != MPI_SUCCESS )
                        return VMPI_ERROR;
                
                if( PMPI_Get_count(&s, MPI_CHAR, &number_of_desc) != MPI_SUCCESS )
                        return VMPI_ERROR;

                if( number_of_desc % sizeof( struct TMP_Partition_desc ) )
                        return VMPI_ERROR;

                number_of_desc /=  sizeof( struct TMP_Partition_desc );

                remote_desc = malloc( number_of_desc * sizeof( struct TMP_Partition_desc ) );

                if( !remote_desc )
                        return VMPI_ERROR;

                /* Receive descs */
                if( PMPI_Recv((void *)remote_desc, number_of_desc * sizeof( struct TMP_Partition_desc), MPI_CHAR, global_rank - 1, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm(), &s) != MPI_SUCCESS )
                        return VMPI_ERROR;

                found = 0;

                /* Check if local CRC is known */
                for( i = 0 ; i < number_of_desc ; i++ )
                {
                        if( remote_desc[i].crc == job_crc )
                        {
                                local_desc.id = remote_desc[i].id;
                                found = 1;
                                break;
                        }
                }
                
                /* We are the first process W this hash */
                if( !found )
                {
                        number_of_desc++;
                        remote_desc = realloc( remote_desc, sizeof( struct TMP_Partition_desc) * number_of_desc );
                        
                        if( !remote_desc )
                                return VMPI_ERROR;
                        
                        /*Pick a new ID and add local desc to remote desc array */
                        local_desc.id = remote_desc[number_of_desc - 2].id + 1;
                        remote_desc[number_of_desc - 1] = local_desc;
                }

                if( global_rank != (global_size - 1 ) )
                {
                        if(PMPI_Send((void *)remote_desc, number_of_desc * sizeof( struct TMP_Partition_desc), MPI_CHAR, global_rank + 1, VMPI_NG_ARBITRARY_VALUE, VMPI_Get_vmpi_comm()) != MPI_SUCCESS )
                                return VMPI_ERROR;
                }

                free( remote_desc );
        }
        
        
        /* The last rank broadcasts the number of partitions and descs*/
        if( PMPI_Bcast( (void *)&number_of_desc, 1, MPI_INT, global_size - 1, VMPI_Get_vmpi_comm() ) != MPI_SUCCESS )
                return VMPI_ERROR;

        


        *partition_count = number_of_desc;

        return local_desc.id;
}

int VMPI_Compatibility_MPMD( int *partition_count )
{
        int rank = 0;
        PMPI_Comm_rank( MPI_COMM_WORLD, &rank );
        int size = 0;
        PMPI_Comm_size( MPI_COMM_WORLD, &size );


        if( size == 1 )
        {
                printf("You must launch at least two processes to perform a splitting\n");
                abort();
        }

        /* Compute process distribution */

        char *proportion = getenv("VMPI_RATIO");
        
        if( !proportion )
        {
                if( rank == 0 )
                {
                        printf("==========================================================");
                        printf("Could not locate the VMPI_RATIO variable in env\n");
                        printf("Please use the dedicated launcher or set it manually\n");
                        printf("==========================================================");
                }
                /* Otherwise consider its 0.5 */
                proportion = "0.5";
        }
        
        errno = 0;
        char *end_ptr = NULL;
        double ratio = strtod( proportion, &end_ptr );
        
        if( (errno == ERANGE ) || (proportion == end_ptr) )
     {
               perror("strtol");
               printf("Could not parse value %s\n", proportion );
               abort();
     }
     
     if( 0.5 < ratio )
     {
                 printf("==========================================================\n");
                 printf("Ratio cannot exceed 0.5 (got %g)\n", ratio );
                 printf("==========================================================\n");
                 abort();
         }

         int analyzer_size = size * ratio;

         if( analyzer_size == 0 )
         {
                 analyzer_size = 1;
         }
        
         int program_size = size - analyzer_size;
         
         if( (program_size + analyzer_size) != size )
         {
                 printf("Failed to compute compatibility MPMD sizes (%d + %d = %d / %d)\n", analyzer_size, program_size, program_size + analyzer_size, size);
                 abort();
         }
         else if( rank == 0 )
         {
                 printf("=================================================================================\n");
                 printf("VMPI will run with %d instrumented processes and %d analyzer (env VMPI_RATIO=%g)\n", program_size, analyzer_size, ratio);
                 printf("=================================================================================\n");
         }
         
        int partition_color = 0;

        if( rank < program_size )
        {
                partition_color = 0;
                //printf("%d is in instrum\n", rank );
        }
        else
        {
                VMPI_Set_partition_name(__vmpi_status.trampoline_partition_name);
                partition_color = 1;
                //printf("%d is in analysis\n", rank );
        }

        /* Setup partition descs */
        *partition_count = 2;

        __vmpi_status.partition_descs = malloc( sizeof( VMPI_Partition_desc ) * 2 );

        if( !__vmpi_status.partition_descs )
        {
                perror("malloc");
                abort();
        }


        return partition_color;
}

void ccg_mpi_error_handler(MPI_Comm *communicator, int *error_code, ...) {
    char error_string[MPI_MAX_ERROR_STRING];
    int error_string_length;
    printf("ccg_mpi_error_handler: entry\n");
    printf("ccg_mpi_error_handler: error_code = %d\n", *error_code);
    MPI_Error_string(*error_code, error_string, &error_string_length);
    error_string[error_string_length] = '\0';
    printf("ccg_mpi_error_handler: error_string = %s\n", error_string);
    printf("ccg_mpi_error_handler: exit\n");
    exit(1);
}


int VMPI_Init(int * argc, char ***argv)
{
        int ret;
        uint64_t crc;

        /* Spawn vmpi's communicator */
        if( PMPI_Comm_dup( MPI_COMM_WORLD , &__vmpi_status.vmpi_communicator ) != MPI_SUCCESS )
                return VMPI_ERROR;




        MPI_Errhandler errhandler;
        PMPI_Comm_create_errhandler(&ccg_mpi_error_handler, &errhandler);
    PMPI_Comm_set_errhandler(__vmpi_status.vmpi_communicator, errhandler);




        if( __vmpi_status.vmpi_trampoline )
        {
                /* Here we do a legacy VMPI splitting (compatibility mode) */
                __vmpi_status.mypartition = VMPI_Compatibility_MPMD( &__vmpi_status.numpartitions );
        }
        else
        {
                /* Hash command line or partition name */
                MALP_Trace_crc64_init();

                if( strlen( __vmpi_status.partition_name ) )
                {
                        crc = MALP_Trace_crc64(__vmpi_status.partition_name, strlen(__vmpi_status.partition_name));
                }
                else
                {
                        crc = VMPI_Compute_job_crc( *argc, *argv );
                }
                
                if( !crc )
                        return VMPI_ERROR;


                /* Retrieve partition ID */
                __vmpi_status.mypartition = VMPI_Detect_Partitions(crc, &__vmpi_status.numpartitions);
        }

        int rank;
        PMPI_Comm_rank( MPI_COMM_WORLD, &rank );

        MPI_Comm tmp_comm;

        /* Split partition communicator according to partition ID */
        ret=PMPI_Comm_split(MPI_COMM_WORLD,__vmpi_status.mypartition, rank, &tmp_comm);

        /* Send partition descriptions to each process */
        VMPI_Dispatch_descs(tmp_comm, *argc, *argv);
        
        /* Store the new communicator in the freshly created partition desc */
        VMPI_Get_desc()->partition_comm = tmp_comm;

        /* If rank 0 display job layout */
        if( rank == 0 )
                VMPI_Display_descs();

        /* Flag VMPI as enabled */
        __vmpi_status.vmpi_enabled = 1;


        /* Here we deroute the execution (if needed) to the trampoline */
        if( __vmpi_status.vmpi_trampoline && (VMPI_Get_partition_id() == 1) )
        {
                (__vmpi_status.vmpi_trampoline)(*argc, *argv);
                VMPI_Release();
                
                PMPI_Finalize();
                exit(0);
        }

        return VMPI_SUCCESS;
}


int VMPI_Release()
{
        PMPI_Barrier( MPI_COMM_WORLD );
        
        __vmpi_status.vmpi_enabled = 0;
        __vmpi_status.numpartitions = 0;

        if( PMPI_Comm_free(&__vmpi_status.partition_descs[__vmpi_status.mypartition].partition_comm) != MPI_SUCCESS )
                return VMPI_ERROR;

        __vmpi_status.mypartition = -1;

        void *to_free = (void *)__vmpi_status.partition_descs;
        __vmpi_status.partition_descs = NULL;
        free( to_free );

        if( PMPI_Comm_free(&__vmpi_status.vmpi_communicator) != MPI_SUCCESS )
                return VMPI_ERROR;

        return VMPI_SUCCESS;
}