From: markls Date: Mon, 30 Jul 2007 11:03:33 +0000 (+0000) Subject: global variables moved into structs and everything compiles. still needs X-Git-Tag: v3.3~1455 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/537b231b47f9c93386dd0aa2e33a5fb1bbb7b7e4 global variables moved into structs and everything compiles. still needs testing. git-svn-id: svn+ssh://scm.gforge.inria.fr/svn/simgrid/simgrid/trunk@3905 48e7efb5-ca39-0410-a469-dd3cf9ba447f --- diff --git a/src/smpi/include/private.h b/src/smpi/include/private.h index 952715b5f0..bbf4d89805 100644 --- a/src/smpi/include/private.h +++ b/src/smpi/include/private.h @@ -1,3 +1,4 @@ +#include "smpi.h" #include "xbt/mallocator.h" #include "xbt/xbt_os_time.h" @@ -37,14 +38,11 @@ typedef struct SMPI_Global { xbt_os_timer_t *timers; smx_mutex_t *timers_mutexes; - int *benchmarking_flags; - smx_mutex_t *benchmarking_flags_mutexes; } s_SMPI_Global_t, *SMPI_Global_t; extern SMPI_Global_t smpi_global; -// smpi_received_message_t struct smpi_received_message_t { smpi_mpi_communicator_t *comm; int src; @@ -52,4 +50,5 @@ struct smpi_received_message_t { int tag; void *buf; }; + typedef struct smpi_received_message_t smpi_received_message_t; diff --git a/src/smpi/include/smpi.h b/src/smpi/include/smpi.h index 2674928710..6165016410 100644 --- a/src/smpi/include/smpi.h +++ b/src/smpi/include/smpi.h @@ -57,7 +57,6 @@ struct smpi_mpi_request_t { short int completed :1; smx_mutex_t mutex; smx_cond_t cond; - xbt_fifo_t waitlist; }; typedef struct smpi_mpi_request_t smpi_mpi_request_t; typedef smpi_mpi_request_t *MPI_Request; @@ -72,23 +71,21 @@ typedef smpi_mpi_op_t *MPI_Op; // global SMPI data structure typedef struct SMPI_MPI_Global { - smpi_mpi_communicator_t mpi_comm_world; + smpi_mpi_communicator_t *mpi_comm_world; - smpi_mpi_status_t mpi_status_ignore; + smpi_mpi_datatype_t *mpi_byte; + smpi_mpi_datatype_t *mpi_int; + smpi_mpi_datatype_t *mpi_double; - smpi_mpi_datatype_t mpi_byte; - smpi_mpi_datatype_t mpi_int; - smpi_mpi_datatype_t mpi_double; - - smpi_mpi_op_t mpi_land; - smpi_mpi_op_t mpi_sum; + smpi_mpi_op_t *mpi_land; + smpi_mpi_op_t *mpi_sum; } s_SMPI_MPI_Global_t, *SMPI_MPI_Global_t; -exterm SMPI_MPI_Global_t smpi_mpi_global; +extern SMPI_MPI_Global_t smpi_mpi_global; #define MPI_COMM_WORLD (smpi_mpi_global->mpi_comm_world) -#define MPI_STATUS_IGNORE (smpi_mpi_global->mpi_status_ignore) +#define MPI_STATUS_IGNORE NULL #define MPI_BYTE (smpi_mpi_global->mpi_byte) #define MPI_DOUBLE (smpi_mpi_global->mpi_double) diff --git a/src/smpi/src/smpi_base.c b/src/smpi/src/smpi_base.c index 25e082d2cc..2b9b8d809c 100644 --- a/src/smpi/src/smpi_base.c +++ b/src/smpi/src/smpi_base.c @@ -3,7 +3,6 @@ #include #include "private.h" -#include "smpi.h" SMPI_Global_t smpi_global = NULL; @@ -38,6 +37,8 @@ int inline smpi_mpi_comm_world_rank_self() int smpi_sender(int argc, char **argv) { + smx_mutex_t mutex; + smx_cond_t cond; smx_process_t self; smx_host_t shost; int rank; @@ -58,8 +59,10 @@ int smpi_sender(int argc, char **argv) int drank; - smx_process_t waitproc; + smx_process_t receiver_process; + mutex = SIMIX_mutex_init(); + cond = SIMIX_cond_init(); self = SIMIX_process_self(); shost = SIMIX_host_self(); rank = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, shost); @@ -103,10 +106,12 @@ int smpi_sender(int argc, char **argv) communicate_action = SIMIX_action_communicate(shost, dhost, "communication", request->datatype->size * request->count * 1.0, -1.0); - SIMIX_register_condition_to_action(communicate_action, request->cond); - SIMIX_register_action_to_condition(communicate_action, request->cond); + SIMIX_register_condition_to_action(communicate_action, cond); + SIMIX_register_action_to_condition(communicate_action, cond); - SIMIX_cond_wait(request->cond, request->mutex); + SIMIX_mutex_lock(mutex); + SIMIX_cond_wait(cond, mutex); + SIMIX_mutex_unlock(mutex); // copy request to appropriate received queue message = xbt_mallocator_get(smpi_global->message_mallocator); @@ -125,16 +130,17 @@ int smpi_sender(int argc, char **argv) request->completed = 1; - // wake up receiver, then any waiting sender - waitproc = smpi_receiver_processes[drank]; - - do { - if (SIMIX_process_is_suspended(waitproc)) { - SIMIX_process_resume(waitproc); - } - } while(waitproc = xbt_fifo_shift(request->waitlist)); + SIMIX_cond_broadcast(request->cond); SIMIX_mutex_unlock(request->mutex); + + // wake up receiver if necessary + receiver_process = smpi_global->receiver_processes[drank]; + + if (SIMIX_process_is_suspended(receiver_process)) { + SIMIX_process_resume(receiver_process); + } + } SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); @@ -145,11 +151,16 @@ int smpi_sender(int argc, char **argv) SIMIX_mutex_lock(smpi_global->start_stop_mutex); smpi_global->ready_process_count--; - if (smpi_global->ready_process_count <= 0) { + if (smpi_global->ready_process_count == 0) { SIMIX_cond_broadcast(smpi_global->start_stop_cond); + } else if (smpi_global->ready_process_count < 0) { + // FIXME: can't happen! abort! } SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + SIMIX_mutex_destroy(mutex); + SIMIX_cond_destroy(cond); + return 0; } @@ -164,7 +175,7 @@ int smpi_receiver(int argc, char **argv) smx_mutex_t message_queue_mutex; int size; - int running_hosts; + int running_hosts_count; smpi_mpi_request_t *request; smpi_received_message_t *message; @@ -178,36 +189,31 @@ int smpi_receiver(int argc, char **argv) rank = smpi_mpi_comm_world_rank_self(); // make sure root is done before own initialization - SIMIX_mutex_lock(init_mutex); - if (!smpi_root_ready) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (!smpi_global->root_ready) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); - request_queue = smpi_pending_recv_requests[rank]; - request_queue_mutex = smpi_pending_recv_requests_mutex[rank]; - message_queue = smpi_received_messages[rank]; - message_queue_mutex = smpi_received_messages_mutex[rank]; - size = smpi_mpi_comm_size(&smpi_mpi_comm_world); + request_queue = smpi_global->pending_recv_request_queues[rank]; + request_queue_mutex = smpi_global->pending_recv_request_queues_mutexes[rank]; + message_queue = smpi_global->received_message_queues[rank]; + message_queue_mutex = smpi_global->received_message_queues_mutexes[rank]; + size = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world); - smpi_receiver_processes[rank] = self; + smpi_global->receiver_processes[rank] = self; // wait for all nodes to signal initializatin complete - SIMIX_mutex_lock(init_mutex); - smpi_ready_count++; - if (smpi_ready_count < 3 * size) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count++; + if (smpi_global->ready_process_count < 3 * size) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } else { - SIMIX_cond_broadcast(init_cond); + SIMIX_cond_broadcast(smpi_global->start_stop_cond); } - SIMIX_mutex_unlock(init_mutex); - - SIMIX_mutex_lock(smpi_running_hosts_mutex); - running_hosts = smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); - - while (0 < running_hosts) { + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + do { request = NULL; message = NULL; @@ -242,37 +248,182 @@ stopsearch: } else { SIMIX_mutex_lock(request->mutex); - memcpy(request->buf, message->buf, request->count * request->datatype->size); + memcpy(request->buf, message->buf, request->datatype->size * request->count); request->src = message->src; request->completed = 1; - - while (waitproc = xbt_fifo_shift(request->waitlist)) { - if (SIMIX_process_is_suspended(waitproc)) { - SIMIX_process_resume(waitproc); - } - } + SIMIX_broadcast(request->cond); SIMIX_mutex_unlock(request->mutex); xbt_free(message->buf); - xbt_mallocator_release(smpi_message_mallocator, message); + xbt_mallocator_release(smpi_global->message_mallocator, message); } - SIMIX_mutex_lock(smpi_running_hosts_mutex); - running_hosts = smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); - } + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + running_hosts_count = smpi_global->running_hosts_count; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); + + } while (0 < running_hosts_count); - SIMIX_mutex_lock(init_mutex); - smpi_ready_count--; - if (smpi_ready_count <= 0) { - SIMIX_cond_broadcast(init_cond); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count--; + if (smpi_global->ready_process_count == 0) { + SIMIX_cond_broadcast(smpi_global->start_stop_cond); + } else if (smpi_global->ready_process_count < 0) { + // FIXME: can't happen, abort! } - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); return 0; } +void *smpi_request_new() +{ + smpi_mpi_request_t *request = xbt_new(smpi_mpi_request_t, 1); + + request->completed = 0; + request->mutex = SIMIX_mutex_init(); + request->cond = SIMIX_cond_init(); + + return request; +} + +void smpi_request_free(void *pointer) { + + smpi_mpi_request_t *request = pointer; + + if (NULL != request) { + SIMIX_cond_destroy(request->cond); + SIMIX_mutex_destroy(request->mutex); + xbt_free(request); + } + + return; +} + +void smpi_request_reset(void *pointer) { + return; +} + + +void *smpi_message_new() +{ + return xbt_new(smpi_received_message_t, 1); +} + +void smpi_message_free(void *pointer) +{ + if (NULL != pointer) { + xbt_free(pointer); + } + + return; +} + +void smpi_message_reset(void *pointer) +{ + return; +} + +void smpi_global_init() +{ + int i; + + int size = SIMIX_host_get_number(); + + smpi_global = xbt_new(s_SMPI_Global_t, 1); + + // config variable + smpi_global->reference_speed = SMPI_DEFAULT_SPEED; + + smpi_global->root_ready = 0; + smpi_global->ready_process_count = 0; + + // start/stop + smpi_global->start_stop_mutex = SIMIX_mutex_init(); + smpi_global->start_stop_cond = SIMIX_cond_init(); + + // processes + smpi_global->sender_processes = xbt_new(smx_process_t, size); + smpi_global->receiver_processes = xbt_new(smx_process_t, size); + + // running hosts + smpi_global->running_hosts_count_mutex = SIMIX_mutex_init(); + smpi_global->running_hosts_count = 0; + + // mallocators + smpi_global->request_mallocator = xbt_mallocator_new(SMPI_REQUEST_MALLOCATOR_SIZE, + smpi_request_new, smpi_request_free, smpi_request_reset); + smpi_global->message_mallocator = xbt_mallocator_new(SMPI_MESSAGE_MALLOCATOR_SIZE, + smpi_message_new, smpi_message_free, smpi_message_reset); + smpi_global->pending_send_request_queues = xbt_new(xbt_fifo_t, size); + smpi_global->pending_send_request_queues_mutexes = xbt_new(smx_mutex_t, size); + smpi_global->pending_recv_request_queues = xbt_new(xbt_fifo_t, size); + smpi_global->pending_recv_request_queues_mutexes = xbt_new(smx_mutex_t, size); + smpi_global->received_message_queues = xbt_new(xbt_fifo_t, size); + smpi_global->received_message_queues_mutexes = xbt_new(smx_mutex_t, size); + smpi_global->timers = xbt_new(xbt_os_timer_t, size); + smpi_global->timers_mutexes = xbt_new(smx_mutex_t, size); + + + for(i = 0; i < size; i++) { + smpi_global->pending_send_request_queues[i] = xbt_fifo_new(); + smpi_global->pending_send_request_queues_mutexes[i] = SIMIX_mutex_init(); + smpi_global->pending_recv_request_queues[i] = xbt_fifo_new(); + smpi_global->pending_recv_request_queues_mutexes[i] = SIMIX_mutex_init(); + smpi_global->received_message_queues[i] = xbt_fifo_new(); + smpi_global->received_message_queues_mutexes[i] = SIMIX_mutex_init(); + smpi_global->timers[i] = xbt_os_timer_new(); + smpi_global->timers_mutexes[i] = SIMIX_mutex_init(); + } + + +} + +void smpi_global_destroy() +{ + int i; + + int size = SIMIX_host_get_number(); + + // start/stop + SIMIX_mutex_destroy(smpi_global->start_stop_mutex); + SIMIX_cond_destroy(smpi_global->start_stop_cond); + + // processes + xbt_free(smpi_global->sender_processes); + xbt_free(smpi_global->receiver_processes); + + // running hosts + SIMIX_mutex_destroy(smpi_global->running_hosts_count_mutex); + + // mallocators + xbt_mallocator_free(smpi_global->request_mallocator); + xbt_mallocator_free(smpi_global->message_mallocator); + + for(i = 0; i < size; i++) { + xbt_fifo_free(smpi_global->pending_send_request_queues[i]); + SIMIX_mutex_destroy(smpi_global->pending_send_request_queues_mutexes[i]); + xbt_fifo_free(smpi_global->pending_recv_request_queues[i]); + SIMIX_mutex_destroy(smpi_global->pending_recv_request_queues_mutexes[i]); + xbt_fifo_free(smpi_global->received_message_queues[i]); + SIMIX_mutex_destroy(smpi_global->received_message_queues_mutexes[i]); + xbt_os_timer_free(smpi_global->timers[i]); + SIMIX_mutex_destroy(smpi_global->timers_mutexes[i]); + } + + xbt_free(smpi_global->pending_send_request_queues); + xbt_free(smpi_global->pending_send_request_queues_mutexes); + xbt_free(smpi_global->pending_recv_request_queues); + xbt_free(smpi_global->pending_recv_request_queues_mutexes); + xbt_free(smpi_global->received_message_queues); + xbt_free(smpi_global->received_message_queues_mutexes); + xbt_free(smpi_global->timers); + xbt_free(smpi_global->timers_mutexes); + + xbt_free(smpi_global); +} + int smpi_run_simulation(int argc, char **argv) { smx_cond_t cond = NULL; @@ -285,9 +436,8 @@ int smpi_run_simulation(int argc, char **argv) SIMIX_global_init(&argc, argv); - // important globals - init_mutex = SIMIX_mutex_init(); - init_cond = SIMIX_cond_init(); + // initialize smpi_global + smpi_global_init(); SIMIX_function_register("smpi_simulated_main", smpi_simulated_main); SIMIX_function_register("smpi_sender", smpi_sender); @@ -327,6 +477,8 @@ int smpi_run_simulation(int argc, char **argv) INFO1("simulation time %g", SIMIX_get_clock()); + smpi_global_destroy(); + SIMIX_clean(); return 0; @@ -342,249 +494,168 @@ void smpi_mpi_sum_func(void *x, void *y, void *z) *(int *)z = *(int *)x + *(int *)y; } -// FIXME: init conds, etc... -void *smpi_new_request() -{ - return xbt_new(smpi_mpi_request_t, 1); -} - -void smpi_free_request(void *pointer) { - smpi_mpi_request_t *request = pointer; - - if (NULL != request) { - SIMIX_mutex_destroy(request->mutex); - SIMIX_cond_destroy(request->cond); - xbt_fifo_free(request->waitlist); - xbt_free(request); - } - - return; -} - -// FIXME: don't keep creating new mutexes... -// flush waitlist instead of new? -void smpi_reset_request(void *pointer) { - smpi_mpi_request_t *request = pointer; - - if (NULL != request) { - request->mutex = SIMIX_mutex_init(); - request->cond = SIMIX_cond_init(); - request->waitlist = xbt_fifo_new(); - // FIXME: clear waitlist - } - - return; -} - - -void *smpi_new_message() -{ - return xbt_new(smpi_received_message_t, 1); -} - -void smpi_do_nothing(void *pointer) -{ - return; -} void smpi_mpi_init() { int i; - int size; smx_process_t process; - smx_host_t *hosts; smx_host_t host; + smx_host_t *hosts; + int size; double duration; + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + smpi_global->running_hosts_count++; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); + // initialize some local variables - host = SIMIX_host_self(); - hosts = SIMIX_host_get_table(); - size = SIMIX_host_get_number(); + process = SIMIX_process_self(); + host = SIMIX_host_self(); + hosts = SIMIX_host_get_table(); + size = SIMIX_host_get_number(); // node 0 sets the globals if (host == hosts[0]) { - // processes - smpi_sender_processes = xbt_new(smx_process_t, size); - smpi_receiver_processes = xbt_new(smx_process_t, size); - - // running hosts - smpi_running_hosts_mutex = SIMIX_mutex_init(); - smpi_running_hosts = size; + smpi_mpi_global = xbt_new(s_SMPI_MPI_Global_t, 1); // global communicator - smpi_mpi_comm_world.size = size; - smpi_mpi_comm_world.barrier = 0; - smpi_mpi_comm_world.barrier_mutex = SIMIX_mutex_init(); - smpi_mpi_comm_world.barrier_cond = SIMIX_cond_init(); - smpi_mpi_comm_world.hosts = hosts; - smpi_mpi_comm_world.processes = xbt_new(smx_process_t, size); - smpi_mpi_comm_world.processes[0] = SIMIX_process_self(); + smpi_mpi_global->mpi_comm_world = xbt_new(smpi_mpi_communicator_t, 1); + smpi_mpi_global->mpi_comm_world->size = size; + smpi_mpi_global->mpi_comm_world->barrier_count = 0; + smpi_mpi_global->mpi_comm_world->barrier_mutex = SIMIX_mutex_init(); + smpi_mpi_global->mpi_comm_world->barrier_cond = SIMIX_cond_init(); + smpi_mpi_global->mpi_comm_world->hosts = hosts; + smpi_mpi_global->mpi_comm_world->processes = xbt_new(smx_process_t, size); + smpi_mpi_global->mpi_comm_world->processes[0] = process; // mpi datatypes - smpi_mpi_byte.size = (size_t)1; - smpi_mpi_int.size = sizeof(int); - smpi_mpi_double.size = sizeof(double); + smpi_mpi_global->mpi_byte = xbt_new(smpi_mpi_datatype_t, 1); + smpi_mpi_global->mpi_byte->size = (size_t)1; + smpi_mpi_global->mpi_int = xbt_new(smpi_mpi_datatype_t, 1); + smpi_mpi_global->mpi_int->size = sizeof(int); + smpi_mpi_global->mpi_double = xbt_new(smpi_mpi_datatype_t, 1); + smpi_mpi_global->mpi_double->size = sizeof(double); // mpi operations - smpi_mpi_land.func = &smpi_mpi_land_func; - smpi_mpi_sum.func = &smpi_mpi_sum_func; - - // smpi globals - smpi_request_mallocator = xbt_mallocator_new(SMPI_REQUEST_MALLOCATOR_SIZE, smpi_new_request, smpi_free_request, smpi_reset_request); - smpi_message_mallocator = xbt_mallocator_new(SMPI_MESSAGE_MALLOCATOR_SIZE, smpi_new_message, smpi_free_message, smpi_do_nothing); - smpi_pending_send_requests = xbt_new(xbt_fifo_t, size); - smpi_pending_send_requests_mutex = xbt_new(smx_mutex_t, size); - smpi_pending_recv_requests = xbt_new(xbt_fifo_t, size); - smpi_pending_recv_requests_mutex = xbt_new(smx_mutex_t, size); - smpi_received_messages = xbt_new(xbt_fifo_t, size); - smpi_received_messages_mutex = xbt_new(smx_mutex_t, size); - - for(i = 0; i < size; i++) { - smpi_pending_send_requests[i] = xbt_fifo_new(); - smpi_pending_send_requests_mutex[i] = SIMIX_mutex_init(); - smpi_pending_recv_requests[i] = xbt_fifo_new(); - smpi_pending_recv_requests_mutex[i] = SIMIX_mutex_init(); - smpi_received_messages[i] = xbt_fifo_new(); - smpi_received_messages_mutex[i] = SIMIX_mutex_init(); - } - - smpi_timer = xbt_os_timer_new(); - smpi_reference_speed = SMPI_DEFAULT_SPEED; - smpi_benchmarking = 0; - smpi_benchmarking_mutex = SIMIX_mutex_init(); + smpi_mpi_global->mpi_land = xbt_new(smpi_mpi_op_t, 1); + smpi_mpi_global->mpi_land->func = smpi_mpi_land_func; + smpi_mpi_global->mpi_sum = xbt_new(smpi_mpi_op_t, 1); + smpi_mpi_global->mpi_sum->func = smpi_mpi_sum_func; // signal all nodes to perform initialization - SIMIX_mutex_lock(init_mutex); - smpi_root_ready = 1; - SIMIX_cond_broadcast(init_cond); - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->root_ready = 1; + SIMIX_cond_broadcast(smpi_global->start_stop_cond); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); } else { // make sure root is done before own initialization - SIMIX_mutex_lock(init_mutex); - if (!smpi_root_ready) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (!smpi_global->root_ready) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } - SIMIX_mutex_unlock(init_mutex); - - smpi_mpi_comm_world.processes[smpi_mpi_comm_rank_self(&smpi_mpi_comm_world)] = SIMIX_process_self(); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + smpi_mpi_global->mpi_comm_world->processes[smpi_mpi_comm_world_rank_self()] = process; } // wait for all nodes to signal initializatin complete - SIMIX_mutex_lock(init_mutex); - smpi_ready_count++; - if (smpi_ready_count < 3 * size) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count++; + if (smpi_global->ready_process_count < 3 * size) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } else { - SIMIX_cond_broadcast(init_cond); + SIMIX_cond_broadcast(smpi_global->start_stop_cond); } - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + return; } void smpi_mpi_finalize() { int i; - SIMIX_mutex_lock(smpi_running_hosts_mutex); - i = --smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + i = --smpi_global->running_hosts_count; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); - SIMIX_mutex_lock(init_mutex); - smpi_ready_count--; - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count--; + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); if (0 >= i) { // wake up senders/receivers - for (i = 0; i < smpi_mpi_comm_world.size; i++) { - if (SIMIX_process_is_suspended(smpi_sender_processes[i])) { - SIMIX_process_resume(smpi_sender_processes[i]); + for (i = 0; i < smpi_mpi_global->mpi_comm_world->size; i++) { + if (SIMIX_process_is_suspended(smpi_global->sender_processes[i])) { + SIMIX_process_resume(smpi_global->sender_processes[i]); } - if (SIMIX_process_is_suspended(smpi_receiver_processes[i])) { - SIMIX_process_resume(smpi_receiver_processes[i]); + if (SIMIX_process_is_suspended(smpi_global->receiver_processes[i])) { + SIMIX_process_resume(smpi_global->receiver_processes[i]); } } // wait for senders/receivers to exit... - SIMIX_mutex_lock(init_mutex); - if (smpi_ready_count > 0) { - SIMIX_cond_wait(init_cond, init_mutex); - } - SIMIX_mutex_unlock(init_mutex); - - SIMIX_mutex_destroy(init_mutex); - SIMIX_cond_destroy(init_cond); - SIMIX_mutex_destroy(smpi_running_hosts_mutex); - - for (i = 0 ; i < smpi_mpi_comm_world.size; i++) { - xbt_fifo_free(smpi_pending_send_requests[i]); - SIMIX_mutex_destroy(smpi_pending_send_requests_mutex[i]); - xbt_fifo_free(smpi_pending_recv_requests[i]); - SIMIX_mutex_destroy(smpi_pending_recv_requests_mutex[i]); - xbt_fifo_free(smpi_received_messages[i]); - SIMIX_mutex_destroy(smpi_received_messages_mutex[i]); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (smpi_global->ready_process_count > 0) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + + SIMIX_mutex_destroy(smpi_mpi_global->mpi_comm_world->barrier_mutex); + SIMIX_cond_destroy(smpi_mpi_global->mpi_comm_world->barrier_cond); + xbt_free(smpi_mpi_global->mpi_comm_world->processes); + xbt_free(smpi_mpi_global->mpi_comm_world); - xbt_mallocator_free(smpi_request_mallocator); - xbt_mallocator_free(smpi_message_mallocator); - xbt_free(smpi_pending_send_requests); - xbt_free(smpi_pending_send_requests_mutex); - xbt_free(smpi_pending_recv_requests); - xbt_free(smpi_pending_recv_requests_mutex); - xbt_free(smpi_received_messages); - xbt_free(smpi_received_messages_mutex); + xbt_free(smpi_mpi_global->mpi_byte); + xbt_free(smpi_mpi_global->mpi_int); + xbt_free(smpi_mpi_global->mpi_double); - SIMIX_mutex_destroy(smpi_mpi_comm_world.barrier_mutex); - SIMIX_cond_destroy(smpi_mpi_comm_world.barrier_cond); - xbt_free(smpi_mpi_comm_world.processes); + xbt_free(smpi_mpi_global->mpi_land); + xbt_free(smpi_mpi_global->mpi_sum); - xbt_os_timer_free(smpi_timer); + xbt_free(smpi_mpi_global); } } +// FIXME: could cause trouble with multithreaded procs on same host... void smpi_bench_begin() { - SIMIX_mutex_lock(smpi_benchmarking_mutex); - xbt_assert0(!smpi_benchmarking, "Already benchmarking"); - smpi_benchmarking = 1; - SIMIX_mutex_unlock(smpi_benchmarking_mutex); - - xbt_os_timer_start(smpi_timer); - + int rank = smpi_mpi_comm_world_rank_self(); + SIMIX_mutex_lock(smpi_global->timers_mutexes[rank]); + xbt_os_timer_start(smpi_global->timers[rank]); return; } void smpi_bench_end() { + int rank = smpi_mpi_comm_world_rank_self(); double duration; smx_host_t host; smx_action_t compute_action; smx_mutex_t mutex; smx_cond_t cond; - SIMIX_mutex_lock(smpi_benchmarking_mutex); - xbt_assert0(smpi_benchmarking, "Not benchmarking yet"); - smpi_benchmarking = 0; - SIMIX_mutex_lock(smpi_benchmarking_mutex); + xbt_os_timer_stop(smpi_global->timers[rank]); - xbt_os_timer_stop(smpi_timer); + duration = xbt_os_timer_elapsed(smpi_global->timers[rank]); + SIMIX_mutex_unlock(smpi_global->timers_mutexes[rank]); - duration = xbt_os_timer_elapsed(smpi_timer); - host = SIMIX_host_self(); + host = smpi_mpi_global->mpi_comm_world->hosts[rank]; compute_action = SIMIX_action_execute(host, "computation", duration * SMPI_DEFAULT_SPEED); mutex = SIMIX_mutex_init(); cond = SIMIX_cond_init(); - SIMIX_mutex_lock(mutex); SIMIX_register_condition_to_action(compute_action, cond); SIMIX_register_action_to_condition(compute_action, cond); + SIMIX_mutex_lock(mutex); SIMIX_cond_wait(cond, mutex); SIMIX_mutex_unlock(mutex); + SIMIX_mutex_destroy(mutex); SIMIX_cond_destroy(cond); @@ -594,14 +665,12 @@ void smpi_bench_end() } void smpi_barrier(smpi_mpi_communicator_t *comm) { - int i; SIMIX_mutex_lock(comm->barrier_mutex); - comm->barrier++; - if(i < comm->size) { + if(++comm->barrier_count < comm->size) { SIMIX_cond_wait(comm->barrier_cond, comm->barrier_mutex); } else { - comm->barrier = 0; + comm->barrier_count = 0; SIMIX_cond_broadcast(comm->barrier_cond); } SIMIX_mutex_unlock(comm->barrier_mutex); @@ -640,7 +709,7 @@ int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype, } else if (0 > tag) { retval = MPI_ERR_TAG; } else { - *request = xbt_mallocator_get(smpi_request_mallocator); + *request = xbt_mallocator_get(smpi_global->request_mallocator); (*request)->comm = comm; (*request)->src = src; (*request)->dst = dst; @@ -648,10 +717,6 @@ int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype, (*request)->buf = buf; (*request)->count = count; (*request)->datatype = datatype; - (*request)->completed = 0; - (*request)->mutex = SIMIX_mutex_init(); - (*request)->cond = SIMIX_cond_init(); - (*request)->waitlist = xbt_fifo_new(); } return retval; } @@ -659,16 +724,16 @@ int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype, int smpi_isend(smpi_mpi_request_t *request) { int retval = MPI_SUCCESS; - int rank = smpi_mpi_comm_rank_self(&smpi_mpi_comm_world); + int rank = smpi_mpi_comm_world_rank_self(); if (NULL != request) { - SIMIX_mutex_lock(smpi_pending_send_requests_mutex[rank]); - xbt_fifo_push(smpi_pending_send_requests[rank], request); - SIMIX_mutex_unlock(smpi_pending_send_requests_mutex[rank]); + SIMIX_mutex_lock(smpi_global->pending_send_request_queues_mutexes[rank]); + xbt_fifo_push(smpi_global->pending_send_request_queues[rank], request); + SIMIX_mutex_unlock(smpi_global->pending_send_request_queues_mutexes[rank]); } - if (SIMIX_process_is_suspended(smpi_sender_processes[rank])) { - SIMIX_process_resume(smpi_sender_processes[rank]); + if (SIMIX_process_is_suspended(smpi_global->sender_processes[rank])) { + SIMIX_process_resume(smpi_global->sender_processes[rank]); } return retval; @@ -677,16 +742,16 @@ int smpi_isend(smpi_mpi_request_t *request) int smpi_irecv(smpi_mpi_request_t *request) { int retval = MPI_SUCCESS; - int rank = smpi_mpi_comm_rank_self(&smpi_mpi_comm_world); + int rank = smpi_mpi_comm_world_rank_self(); if (NULL != request) { - SIMIX_mutex_lock(smpi_pending_recv_requests_mutex[rank]); - xbt_fifo_push(smpi_pending_recv_requests[rank], request); - SIMIX_mutex_unlock(smpi_pending_recv_requests_mutex[rank]); + SIMIX_mutex_lock(smpi_global->pending_recv_request_queues_mutexes[rank]); + xbt_fifo_push(smpi_global->pending_recv_request_queues[rank], request); + SIMIX_mutex_unlock(smpi_global->pending_recv_request_queues_mutexes[rank]); } - if (SIMIX_process_is_suspended(smpi_receiver_processes[rank])) { - SIMIX_process_resume(smpi_receiver_processes[rank]); + if (SIMIX_process_is_suspended(smpi_global->receiver_processes[rank])) { + SIMIX_process_resume(smpi_global->receiver_processes[rank]); } return retval; @@ -700,18 +765,10 @@ void smpi_wait(smpi_mpi_request_t *request, smpi_mpi_status_t *status) if (NULL != request) { SIMIX_mutex_lock(request->mutex); if (!request->completed) { - xbt_fifo_push(request->waitlist, self); - suspend = 1; + SIMIX_cond_wait(request->cond, request->mutex); } + status->MPI_SOURCE = request->src; SIMIX_mutex_unlock(request->mutex); - if (suspend) { - SIMIX_process_suspend(self); - } - if (NULL != status && MPI_STATUS_IGNORE != status) { - SIMIX_mutex_lock(request->mutex); - status->MPI_SOURCE = request->src; - SIMIX_mutex_unlock(request->mutex); - } } } @@ -744,14 +801,18 @@ unsigned int smpi_sleep(unsigned int seconds) sleep_action = SIMIX_action_sleep(host, seconds); mutex = SIMIX_mutex_init(); cond = SIMIX_cond_init(); - SIMIX_mutex_lock(mutex); + SIMIX_register_condition_to_action(sleep_action, cond); SIMIX_register_action_to_condition(sleep_action, cond); + SIMIX_mutex_lock(mutex); SIMIX_cond_wait(cond, mutex); SIMIX_mutex_unlock(mutex); + SIMIX_mutex_destroy(mutex); SIMIX_cond_destroy(cond); + // FIXME: check for success/failure? + smpi_bench_begin(); return 0; } @@ -759,9 +820,9 @@ unsigned int smpi_sleep(unsigned int seconds) void smpi_exit(int status) { smpi_bench_end(); - SIMIX_mutex_lock(smpi_running_hosts_mutex); - smpi_running_hosts--; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + smpi_global->running_hosts_count--; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); SIMIX_process_kill(SIMIX_process_self()); return; }