X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/1c083f59ec05e40f632911ca16d6180e9e59e58c..0c5fdc16c1cd593d4434110621ca3b6ed81bdec5:/src/smpi/src/smpi_base.c diff --git a/src/smpi/src/smpi_base.c b/src/smpi/src/smpi_base.c index 44698c78c8..7d66cf0a32 100644 --- a/src/smpi/src/smpi_base.c +++ b/src/smpi/src/smpi_base.c @@ -1,57 +1,16 @@ #include - #include #include -#include "xbt/xbt_os_time.h" -#include "xbt/mallocator.h" -#include "smpi.h" - -// FIXME: move globals into structure... - -xbt_mallocator_t smpi_request_mallocator = NULL; -xbt_mallocator_t smpi_message_mallocator = NULL; - -xbt_fifo_t *smpi_pending_send_requests = NULL; -smx_mutex_t *smpi_pending_send_requests_mutex = NULL; - -xbt_fifo_t *smpi_pending_recv_requests = NULL; -smx_mutex_t *smpi_pending_recv_requests_mutex = NULL; - -xbt_fifo_t *smpi_received_messages = NULL; -smx_mutex_t *smpi_received_messages_mutex = NULL; - -smx_process_t *smpi_sender_processes = NULL; -smx_process_t *smpi_receiver_processes = NULL; - -int smpi_running_hosts = 0; - -smpi_mpi_communicator_t smpi_mpi_comm_world; - -smpi_mpi_status_t smpi_mpi_status_ignore; - -smpi_mpi_datatype_t smpi_mpi_byte; -smpi_mpi_datatype_t smpi_mpi_int; -smpi_mpi_datatype_t smpi_mpi_double; -smpi_mpi_op_t smpi_mpi_land; -smpi_mpi_op_t smpi_mpi_sum; +#include "private.h" -static xbt_os_timer_t smpi_timer; -static int smpi_benchmarking; -static double smpi_reference_speed; +SMPI_Global_t smpi_global = NULL; -// mutexes -smx_mutex_t smpi_running_hosts_mutex = NULL; -smx_mutex_t smpi_benchmarking_mutex = NULL; -smx_mutex_t init_mutex = NULL; -smx_cond_t init_cond = NULL; - -int smpi_root_ready = 0; -int smpi_ready_count = 0; +SMPI_MPI_Global_t smpi_mpi_global = NULL; XBT_LOG_NEW_DEFAULT_CATEGORY(smpi, "SMPI"); -int inline smpi_mpi_comm_size(smpi_mpi_communicator_t *comm) +int inline smpi_mpi_comm_size(smpi_mpi_communicator_t *comm) { return comm->size; } @@ -73,7 +32,7 @@ int inline smpi_mpi_comm_rank_self(smpi_mpi_communicator_t *comm) int inline smpi_mpi_comm_world_rank_self() { - return smpi_mpi_comm_rank(&smpi_mpi_comm_world, SIMIX_host_self()); + return smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, SIMIX_host_self()); } int smpi_sender(int argc, char **argv) @@ -81,50 +40,53 @@ int smpi_sender(int argc, char **argv) smx_process_t self; smx_host_t shost; int rank; + xbt_fifo_t request_queue; smx_mutex_t request_queue_mutex; int size; - int running_hosts = 0; + + int running_hosts_count; + smpi_mpi_request_t *request; + smx_host_t dhost; + smx_action_t communicate_action; - smpi_received_message_t *scratch; + + smpi_received_message_t *message; + int drank; - smx_process_t waitproc; + + smx_process_t receiver_process; self = SIMIX_process_self(); shost = SIMIX_host_self(); - rank = smpi_mpi_comm_rank(&smpi_mpi_comm_world, shost); + rank = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, shost); // make sure root is done before own initialization - SIMIX_mutex_lock(init_mutex); - if (!smpi_root_ready) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (!smpi_global->root_ready) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } - SIMIX_mutex_unlock(init_mutex); - - request_queue = smpi_pending_send_requests[rank]; - request_queue_mutex = smpi_pending_send_requests_mutex[rank]; + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); - size = smpi_mpi_comm_size(&smpi_mpi_comm_world); + request_queue = smpi_global->pending_send_request_queues[rank]; + request_queue_mutex = smpi_global->pending_send_request_queues_mutexes[rank]; + size = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world); - smpi_sender_processes[rank] = self; + smpi_global->sender_processes[rank] = self; // wait for all nodes to signal initializatin complete - SIMIX_mutex_lock(init_mutex); - smpi_ready_count++; - if (smpi_ready_count < 3 * size) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count++; + if (smpi_global->ready_process_count < 3 * size) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } else { - SIMIX_cond_broadcast(init_cond); + SIMIX_cond_broadcast(smpi_global->start_stop_cond); } - SIMIX_mutex_unlock(init_mutex); - - SIMIX_mutex_lock(smpi_running_hosts_mutex); - running_hosts = smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); - while (0 < running_hosts) { + do { SIMIX_mutex_lock(request_queue_mutex); request = xbt_fifo_shift(request_queue); @@ -133,11 +95,27 @@ int smpi_sender(int argc, char **argv) if (NULL == request) { SIMIX_process_suspend(self); } else { + SIMIX_mutex_lock(request->mutex); + // copy request to appropriate received queue + message = xbt_mallocator_get(smpi_global->message_mallocator); + message->comm = request->comm; + message->src = request->src; + message->dst = request->dst; + message->tag = request->tag; + message->buf = xbt_malloc(request->datatype->size * request->count); + memcpy(message->buf, request->buf, request->datatype->size * request->count); + dhost = request->comm->hosts[request->dst]; + drank = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, dhost); + + SIMIX_mutex_lock(smpi_global->received_message_queues_mutexes[drank]); + xbt_fifo_push(smpi_global->received_message_queues[drank], message); + SIMIX_mutex_unlock(smpi_global->received_message_queues_mutexes[drank]); + + request->completed = 1; - // FIXME: not at all sure I can assume magic just happens here.... communicate_action = SIMIX_action_communicate(shost, dhost, "communication", request->datatype->size * request->count * 1.0, -1.0); @@ -146,36 +124,31 @@ int smpi_sender(int argc, char **argv) SIMIX_cond_wait(request->cond, request->mutex); - // copy request to appropriate received queue - scratch = xbt_mallocator_get(smpi_message_mallocator); - scratch->comm = request->comm; - scratch->src = request->src; - scratch->dst = request->dst; - scratch->tag = request->tag; - scratch->buf = request->buf; - drank = smpi_mpi_comm_rank(&smpi_mpi_comm_world, dhost); - SIMIX_mutex_lock(smpi_received_messages_mutex[drank]); - xbt_fifo_push(smpi_received_messages[drank], scratch); - SIMIX_mutex_unlock(smpi_received_messages_mutex[drank]); - - request->completed = 1; + SIMIX_mutex_unlock(request->mutex); - // wake up receiver, then any waiting sender - waitproc = smpi_receiver_processes[drank]; + // wake up receiver if necessary + receiver_process = smpi_global->receiver_processes[drank]; - do { - if (SIMIX_process_is_suspended(waitproc)) { - SIMIX_process_resume(waitproc); - } - } while(waitproc = xbt_fifo_shift(request->waitlist)); + if (SIMIX_process_is_suspended(receiver_process)) { + SIMIX_process_resume(receiver_process); + } - SIMIX_mutex_unlock(request->mutex); } - SIMIX_mutex_lock(smpi_running_hosts_mutex); - running_hosts = smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + running_hosts_count = smpi_global->running_hosts_count; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); + + } while (0 < running_hosts_count); + + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count--; + if (smpi_global->ready_process_count == 0) { + SIMIX_cond_broadcast(smpi_global->start_stop_cond); + } else if (smpi_global->ready_process_count < 0) { + // FIXME: can't happen! abort! } + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); return 0; } @@ -184,51 +157,54 @@ int smpi_receiver(int argc, char **argv) { smx_process_t self; int rank; + xbt_fifo_t request_queue; smx_mutex_t request_queue_mutex; xbt_fifo_t message_queue; smx_mutex_t message_queue_mutex; int size; - int running_hosts; - xbt_fifo_item_t request_item, message_item; + + int running_hosts_count; + smpi_mpi_request_t *request; smpi_received_message_t *message; + + xbt_fifo_item_t request_item; + xbt_fifo_item_t message_item; + smx_process_t waitproc; self = SIMIX_process_self(); rank = smpi_mpi_comm_world_rank_self(); // make sure root is done before own initialization - SIMIX_mutex_lock(init_mutex); - if (!smpi_root_ready) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (!smpi_global->root_ready) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } - SIMIX_mutex_unlock(init_mutex); - - request_queue = smpi_pending_recv_requests[rank]; - request_queue_mutex = smpi_pending_recv_requests_mutex[rank]; + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); - message_queue = smpi_received_messages[rank]; - message_queue_mutex = smpi_received_messages_mutex[rank]; + request_queue = smpi_global->pending_recv_request_queues[rank]; + request_queue_mutex = smpi_global->pending_recv_request_queues_mutexes[rank]; + message_queue = smpi_global->received_message_queues[rank]; + message_queue_mutex = smpi_global->received_message_queues_mutexes[rank]; + size = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world); - size = smpi_mpi_comm_size(&smpi_mpi_comm_world); - smpi_receiver_processes[rank] = self; + smpi_global->receiver_processes[rank] = self; // wait for all nodes to signal initializatin complete - SIMIX_mutex_lock(init_mutex); - smpi_ready_count++; - if (smpi_ready_count < 3 * size) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count++; + if (smpi_global->ready_process_count < 3 * size) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } else { - SIMIX_cond_broadcast(init_cond); + SIMIX_cond_broadcast(smpi_global->start_stop_cond); } - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); - SIMIX_mutex_lock(smpi_running_hosts_mutex); - running_hosts = smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); - - while (0 < running_hosts) { + do { + request = NULL; + message = NULL; // FIXME: better algorithm, maybe some kind of balanced tree? or a heap? @@ -260,28 +236,183 @@ stopsearch: SIMIX_process_suspend(self); } else { SIMIX_mutex_lock(request->mutex); - memcpy(request->buf, message->buf, request->count * request->datatype->size); + + memcpy(request->buf, message->buf, request->datatype->size * request->count); request->src = message->src; request->completed = 1; + SIMIX_cond_broadcast(request->cond); - while (waitproc = xbt_fifo_shift(request->waitlist)) { - if (SIMIX_process_is_suspended(waitproc)) { - SIMIX_process_resume(waitproc); - } - } SIMIX_mutex_unlock(request->mutex); - xbt_mallocator_release(smpi_message_mallocator, message); + xbt_free(message->buf); + xbt_mallocator_release(smpi_global->message_mallocator, message); } - SIMIX_mutex_lock(smpi_running_hosts_mutex); - running_hosts = smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + running_hosts_count = smpi_global->running_hosts_count; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); + + } while (0 < running_hosts_count); + + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count--; + if (smpi_global->ready_process_count == 0) { + SIMIX_cond_broadcast(smpi_global->start_stop_cond); + } else if (smpi_global->ready_process_count < 0) { + // FIXME: can't happen, abort! } + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); return 0; } +void *smpi_request_new() +{ + smpi_mpi_request_t *request = xbt_new(smpi_mpi_request_t, 1); + + request->completed = 0; + request->mutex = SIMIX_mutex_init(); + request->cond = SIMIX_cond_init(); + + return request; +} + +void smpi_request_free(void *pointer) { + + smpi_mpi_request_t *request = pointer; + + if (NULL != request) { + SIMIX_cond_destroy(request->cond); + SIMIX_mutex_destroy(request->mutex); + xbt_free(request); + } + + return; +} + +void smpi_request_reset(void *pointer) { + return; +} + + +void *smpi_message_new() +{ + return xbt_new(smpi_received_message_t, 1); +} + +void smpi_message_free(void *pointer) +{ + if (NULL != pointer) { + xbt_free(pointer); + } + + return; +} + +void smpi_message_reset(void *pointer) +{ + return; +} + +void smpi_global_init() +{ + int i; + + int size = SIMIX_host_get_number(); + + smpi_global = xbt_new(s_SMPI_Global_t, 1); + + // config variable + smpi_global->reference_speed = SMPI_DEFAULT_SPEED; + + smpi_global->root_ready = 0; + smpi_global->ready_process_count = 0; + + // start/stop + smpi_global->start_stop_mutex = SIMIX_mutex_init(); + smpi_global->start_stop_cond = SIMIX_cond_init(); + + // processes + smpi_global->sender_processes = xbt_new(smx_process_t, size); + smpi_global->receiver_processes = xbt_new(smx_process_t, size); + + // running hosts + smpi_global->running_hosts_count_mutex = SIMIX_mutex_init(); + smpi_global->running_hosts_count = 0; + + // mallocators + smpi_global->request_mallocator = xbt_mallocator_new(SMPI_REQUEST_MALLOCATOR_SIZE, + smpi_request_new, smpi_request_free, smpi_request_reset); + smpi_global->message_mallocator = xbt_mallocator_new(SMPI_MESSAGE_MALLOCATOR_SIZE, + smpi_message_new, smpi_message_free, smpi_message_reset); + + // + smpi_global->pending_send_request_queues = xbt_new(xbt_fifo_t, size); + smpi_global->pending_send_request_queues_mutexes = xbt_new(smx_mutex_t, size); + smpi_global->pending_recv_request_queues = xbt_new(xbt_fifo_t, size); + smpi_global->pending_recv_request_queues_mutexes = xbt_new(smx_mutex_t, size); + smpi_global->received_message_queues = xbt_new(xbt_fifo_t, size); + smpi_global->received_message_queues_mutexes = xbt_new(smx_mutex_t, size); + smpi_global->timers = xbt_new(xbt_os_timer_t, size); + smpi_global->timers_mutexes = xbt_new(smx_mutex_t, size); + + for(i = 0; i < size; i++) { + smpi_global->pending_send_request_queues[i] = xbt_fifo_new(); + smpi_global->pending_send_request_queues_mutexes[i] = SIMIX_mutex_init(); + smpi_global->pending_recv_request_queues[i] = xbt_fifo_new(); + smpi_global->pending_recv_request_queues_mutexes[i] = SIMIX_mutex_init(); + smpi_global->received_message_queues[i] = xbt_fifo_new(); + smpi_global->received_message_queues_mutexes[i] = SIMIX_mutex_init(); + smpi_global->timers[i] = xbt_os_timer_new(); + smpi_global->timers_mutexes[i] = SIMIX_mutex_init(); + } + +} + +void smpi_global_destroy() +{ + int i; + + int size = SIMIX_host_get_number(); + + // start/stop + SIMIX_mutex_destroy(smpi_global->start_stop_mutex); + SIMIX_cond_destroy(smpi_global->start_stop_cond); + + // processes + xbt_free(smpi_global->sender_processes); + xbt_free(smpi_global->receiver_processes); + + // running hosts + SIMIX_mutex_destroy(smpi_global->running_hosts_count_mutex); + + // mallocators + xbt_mallocator_free(smpi_global->request_mallocator); + xbt_mallocator_free(smpi_global->message_mallocator); + + for(i = 0; i < size; i++) { + xbt_fifo_free(smpi_global->pending_send_request_queues[i]); + SIMIX_mutex_destroy(smpi_global->pending_send_request_queues_mutexes[i]); + xbt_fifo_free(smpi_global->pending_recv_request_queues[i]); + SIMIX_mutex_destroy(smpi_global->pending_recv_request_queues_mutexes[i]); + xbt_fifo_free(smpi_global->received_message_queues[i]); + SIMIX_mutex_destroy(smpi_global->received_message_queues_mutexes[i]); + xbt_os_timer_free(smpi_global->timers[i]); + SIMIX_mutex_destroy(smpi_global->timers_mutexes[i]); + } + + xbt_free(smpi_global->pending_send_request_queues); + xbt_free(smpi_global->pending_send_request_queues_mutexes); + xbt_free(smpi_global->pending_recv_request_queues); + xbt_free(smpi_global->pending_recv_request_queues_mutexes); + xbt_free(smpi_global->received_message_queues); + xbt_free(smpi_global->received_message_queues_mutexes); + xbt_free(smpi_global->timers); + xbt_free(smpi_global->timers_mutexes); + + xbt_free(smpi_global); +} + int smpi_run_simulation(int argc, char **argv) { smx_cond_t cond = NULL; @@ -294,16 +425,20 @@ int smpi_run_simulation(int argc, char **argv) SIMIX_global_init(&argc, argv); - init_mutex = SIMIX_mutex_init(); - init_cond = SIMIX_cond_init(); - SIMIX_function_register("smpi_simulated_main", smpi_simulated_main); SIMIX_function_register("smpi_sender", smpi_sender); SIMIX_function_register("smpi_receiver", smpi_receiver); + + // FIXME: ought to verify these files... SIMIX_create_environment(argv[1]); + + // must initialize globals between creating environment and launching app.... + smpi_global_init(); + SIMIX_launch_application(argv[2]); /* Prepare to display some more info when dying on Ctrl-C pressing */ + // FIXME: doesn't work //signal(SIGINT, inthandler); /* Clean IO before the run */ @@ -326,10 +461,16 @@ int smpi_run_simulation(int argc, char **argv) SIMIX_action_destroy(action); } } + xbt_fifo_free(actions_failed); xbt_fifo_free(actions_done); + INFO1("simulation time %g", SIMIX_get_clock()); + + smpi_global_destroy(); + SIMIX_clean(); + return 0; } @@ -343,201 +484,191 @@ void smpi_mpi_sum_func(void *x, void *y, void *z) *(int *)z = *(int *)x + *(int *)y; } -void *smpi_new_request() -{ - return xbt_new(smpi_mpi_request_t, 1); -} - -void *smpi_new_message() -{ - return xbt_new(smpi_received_message_t, 1); -} void smpi_mpi_init() { int i; - int size; smx_process_t process; - smx_host_t *hosts; smx_host_t host; + smx_host_t *hosts; + int size; double duration; + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + smpi_global->running_hosts_count++; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); + // initialize some local variables - host = SIMIX_host_self(); - hosts = SIMIX_host_get_table(); - size = SIMIX_host_get_number(); + process = SIMIX_process_self(); + host = SIMIX_host_self(); + hosts = SIMIX_host_get_table(); + size = SIMIX_host_get_number(); // node 0 sets the globals if (host == hosts[0]) { - // processes - smpi_sender_processes = xbt_new(smx_process_t, size); - smpi_receiver_processes = xbt_new(smx_process_t, size); - - // running hosts - smpi_running_hosts_mutex = SIMIX_mutex_init(); - smpi_running_hosts = size; + smpi_mpi_global = xbt_new(s_SMPI_MPI_Global_t, 1); // global communicator - smpi_mpi_comm_world.size = size; - smpi_mpi_comm_world.barrier = 0; - smpi_mpi_comm_world.barrier_mutex = SIMIX_mutex_init(); - smpi_mpi_comm_world.barrier_cond = SIMIX_cond_init(); - smpi_mpi_comm_world.hosts = hosts; - smpi_mpi_comm_world.processes = xbt_new(smx_process_t, size); - smpi_mpi_comm_world.processes[0] = SIMIX_process_self(); + smpi_mpi_global->mpi_comm_world = xbt_new(smpi_mpi_communicator_t, 1); + smpi_mpi_global->mpi_comm_world->size = size; + smpi_mpi_global->mpi_comm_world->barrier_count = 0; + smpi_mpi_global->mpi_comm_world->barrier_mutex = SIMIX_mutex_init(); + smpi_mpi_global->mpi_comm_world->barrier_cond = SIMIX_cond_init(); + smpi_mpi_global->mpi_comm_world->hosts = hosts; + smpi_mpi_global->mpi_comm_world->processes = xbt_new(smx_process_t, size); + smpi_mpi_global->mpi_comm_world->processes[0] = process; // mpi datatypes - smpi_mpi_byte.size = (size_t)1; - smpi_mpi_int.size = sizeof(int); - smpi_mpi_double.size = sizeof(double); + smpi_mpi_global->mpi_byte = xbt_new(smpi_mpi_datatype_t, 1); + smpi_mpi_global->mpi_byte->size = (size_t)1; + smpi_mpi_global->mpi_int = xbt_new(smpi_mpi_datatype_t, 1); + smpi_mpi_global->mpi_int->size = sizeof(int); + smpi_mpi_global->mpi_double = xbt_new(smpi_mpi_datatype_t, 1); + smpi_mpi_global->mpi_double->size = sizeof(double); // mpi operations - smpi_mpi_land.func = &smpi_mpi_land_func; - smpi_mpi_sum.func = &smpi_mpi_sum_func; - - // smpi globals - smpi_request_mallocator = xbt_mallocator_new(SMPI_REQUEST_MALLOCATOR_SIZE, &smpi_new_request, &xbt_free, NULL); - smpi_message_mallocator = xbt_mallocator_new(SMPI_MESSAGE_MALLOCATOR_SIZE, &smpi_new_message, &xbt_free, NULL); - smpi_pending_send_requests = xbt_new(xbt_fifo_t, size); - smpi_pending_send_requests_mutex = xbt_new(smx_mutex_t, size); - smpi_pending_recv_requests = xbt_new(xbt_fifo_t, size); - smpi_pending_recv_requests_mutex = xbt_new(smx_mutex_t, size); - smpi_received_messages = xbt_new(xbt_fifo_t, size); - smpi_received_messages_mutex = xbt_new(smx_mutex_t, size); - - for(i = 0; i < size; i++) { - smpi_pending_send_requests[i] = xbt_fifo_new(); - smpi_pending_send_requests_mutex[i] = SIMIX_mutex_init(); - smpi_pending_recv_requests[i] = xbt_fifo_new(); - smpi_pending_recv_requests_mutex[i] = SIMIX_mutex_init(); - smpi_received_messages[i] = xbt_fifo_new(); - smpi_received_messages_mutex[i] = SIMIX_mutex_init(); - } - - smpi_timer = xbt_os_timer_new(); - smpi_reference_speed = SMPI_DEFAULT_SPEED; - smpi_benchmarking = 0; - smpi_benchmarking_mutex = SIMIX_mutex_init(); + smpi_mpi_global->mpi_land = xbt_new(smpi_mpi_op_t, 1); + smpi_mpi_global->mpi_land->func = smpi_mpi_land_func; + smpi_mpi_global->mpi_sum = xbt_new(smpi_mpi_op_t, 1); + smpi_mpi_global->mpi_sum->func = smpi_mpi_sum_func; // signal all nodes to perform initialization - SIMIX_mutex_lock(init_mutex); - smpi_root_ready = 1; - SIMIX_cond_broadcast(init_cond); - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->root_ready = 1; + SIMIX_cond_broadcast(smpi_global->start_stop_cond); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); } else { // make sure root is done before own initialization - SIMIX_mutex_lock(init_mutex); - if (!smpi_root_ready) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (!smpi_global->root_ready) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } - SIMIX_mutex_unlock(init_mutex); - - smpi_mpi_comm_world.processes[smpi_mpi_comm_rank_self(&smpi_mpi_comm_world)] = SIMIX_process_self(); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + smpi_mpi_global->mpi_comm_world->processes[smpi_mpi_comm_world_rank_self()] = process; } // wait for all nodes to signal initializatin complete - SIMIX_mutex_lock(init_mutex); - smpi_ready_count++; - if (smpi_ready_count < 3 * size) { - SIMIX_cond_wait(init_cond, init_mutex); + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count++; + if (smpi_global->ready_process_count < 3 * size) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } else { - SIMIX_cond_broadcast(init_cond); + SIMIX_cond_broadcast(smpi_global->start_stop_cond); } - SIMIX_mutex_unlock(init_mutex); + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + return; } void smpi_mpi_finalize() { int i; - SIMIX_mutex_lock(smpi_running_hosts_mutex); - i = --smpi_running_hosts; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + i = --smpi_global->running_hosts_count; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); + + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + smpi_global->ready_process_count--; + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); if (0 >= i) { - SIMIX_mutex_destroy(smpi_running_hosts_mutex); + // wake up senders/receivers + for (i = 0; i < smpi_mpi_global->mpi_comm_world->size; i++) { + if (SIMIX_process_is_suspended(smpi_global->sender_processes[i])) { + SIMIX_process_resume(smpi_global->sender_processes[i]); + } + if (SIMIX_process_is_suspended(smpi_global->receiver_processes[i])) { + SIMIX_process_resume(smpi_global->receiver_processes[i]); + } + } - for (i = 0 ; i < smpi_mpi_comm_world.size; i++) { - xbt_fifo_free(smpi_pending_send_requests[i]); - SIMIX_mutex_destroy(smpi_pending_send_requests_mutex[i]); - xbt_fifo_free(smpi_pending_recv_requests[i]); - SIMIX_mutex_destroy(smpi_pending_recv_requests_mutex[i]); - xbt_fifo_free(smpi_received_messages[i]); - SIMIX_mutex_destroy(smpi_received_messages_mutex[i]); + // wait for senders/receivers to exit... + SIMIX_mutex_lock(smpi_global->start_stop_mutex); + if (smpi_global->ready_process_count > 0) { + SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex); } + SIMIX_mutex_unlock(smpi_global->start_stop_mutex); + + SIMIX_mutex_destroy(smpi_mpi_global->mpi_comm_world->barrier_mutex); + SIMIX_cond_destroy(smpi_mpi_global->mpi_comm_world->barrier_cond); + xbt_free(smpi_mpi_global->mpi_comm_world->processes); + xbt_free(smpi_mpi_global->mpi_comm_world); - xbt_mallocator_free(smpi_request_mallocator); - xbt_mallocator_free(smpi_message_mallocator); - xbt_free(smpi_pending_send_requests); - xbt_free(smpi_pending_send_requests_mutex); - xbt_free(smpi_pending_recv_requests); - xbt_free(smpi_pending_recv_requests_mutex); - xbt_free(smpi_received_messages); - xbt_free(smpi_received_messages_mutex); + xbt_free(smpi_mpi_global->mpi_byte); + xbt_free(smpi_mpi_global->mpi_int); + xbt_free(smpi_mpi_global->mpi_double); - SIMIX_mutex_destroy(smpi_mpi_comm_world.barrier_mutex); - SIMIX_cond_destroy(smpi_mpi_comm_world.barrier_cond); - xbt_free(smpi_mpi_comm_world.processes); + xbt_free(smpi_mpi_global->mpi_land); + xbt_free(smpi_mpi_global->mpi_sum); - xbt_os_timer_free(smpi_timer); + xbt_free(smpi_mpi_global); } } +// FIXME: could cause trouble with multithreaded procs on same host... void smpi_bench_begin() { - xbt_assert0(!smpi_benchmarking, "Already benchmarking"); - smpi_benchmarking = 1; - xbt_os_timer_start(smpi_timer); + int rank = smpi_mpi_comm_world_rank_self(); + SIMIX_mutex_lock(smpi_global->timers_mutexes[rank]); + xbt_os_timer_start(smpi_global->timers[rank]); return; } void smpi_bench_end() { + int rank = smpi_mpi_comm_world_rank_self(); double duration; smx_host_t host; smx_action_t compute_action; smx_mutex_t mutex; smx_cond_t cond; - xbt_assert0(smpi_benchmarking, "Not benchmarking yet"); - smpi_benchmarking = 0; - xbt_os_timer_stop(smpi_timer); - duration = xbt_os_timer_elapsed(smpi_timer); - host = SIMIX_host_self(); + xbt_os_timer_stop(smpi_global->timers[rank]); + + duration = xbt_os_timer_elapsed(smpi_global->timers[rank]); + SIMIX_mutex_unlock(smpi_global->timers_mutexes[rank]); + + host = smpi_mpi_global->mpi_comm_world->hosts[rank]; compute_action = SIMIX_action_execute(host, "computation", duration * SMPI_DEFAULT_SPEED); mutex = SIMIX_mutex_init(); cond = SIMIX_cond_init(); - SIMIX_mutex_lock(mutex); + SIMIX_register_condition_to_action(compute_action, cond); SIMIX_register_action_to_condition(compute_action, cond); + SIMIX_mutex_lock(mutex); SIMIX_cond_wait(cond, mutex); SIMIX_mutex_unlock(mutex); + SIMIX_mutex_destroy(mutex); SIMIX_cond_destroy(cond); + // FIXME: check for success/failure? + return; } void smpi_barrier(smpi_mpi_communicator_t *comm) { - int i; + SIMIX_mutex_lock(comm->barrier_mutex); - comm->barrier++; - if(i < comm->size) { + if(++comm->barrier_count < comm->size) { SIMIX_cond_wait(comm->barrier_cond, comm->barrier_mutex); } else { - comm->barrier = 0; + comm->barrier_count = 0; SIMIX_cond_broadcast(comm->barrier_cond); } SIMIX_mutex_unlock(comm->barrier_mutex); + + return; } +// FIXME: smarter algorithm... int smpi_comm_rank(smpi_mpi_communicator_t *comm, smx_host_t host) { int i; @@ -568,7 +699,7 @@ int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype, } else if (0 > tag) { retval = MPI_ERR_TAG; } else { - *request = xbt_mallocator_get(smpi_request_mallocator); + *request = xbt_mallocator_get(smpi_global->request_mallocator); (*request)->comm = comm; (*request)->src = src; (*request)->dst = dst; @@ -576,61 +707,60 @@ int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype, (*request)->buf = buf; (*request)->count = count; (*request)->datatype = datatype; - (*request)->completed = 0; - (*request)->mutex = SIMIX_mutex_init(); - (*request)->cond = SIMIX_cond_init(); - (*request)->waitlist = NULL; } return retval; } int smpi_isend(smpi_mpi_request_t *request) { - int rank = smpi_mpi_comm_rank_self(&smpi_mpi_comm_world); + int retval = MPI_SUCCESS; + int rank = smpi_mpi_comm_world_rank_self(); - SIMIX_mutex_lock(smpi_pending_send_requests_mutex[rank]); - xbt_fifo_push(smpi_pending_send_requests[rank], request); - SIMIX_mutex_unlock(smpi_pending_send_requests_mutex[rank]); + if (NULL != request) { + SIMIX_mutex_lock(smpi_global->pending_send_request_queues_mutexes[rank]); + xbt_fifo_push(smpi_global->pending_send_request_queues[rank], request); + SIMIX_mutex_unlock(smpi_global->pending_send_request_queues_mutexes[rank]); + } - if (MSG_process_is_suspended(smpi_sender_processes[rank])) { - MSG_process_resume(smpi_sender_processes[rank]); + if (SIMIX_process_is_suspended(smpi_global->sender_processes[rank])) { + SIMIX_process_resume(smpi_global->sender_processes[rank]); } + + return retval; } int smpi_irecv(smpi_mpi_request_t *request) { - int rank = smpi_mpi_comm_rank_self(&smpi_mpi_comm_world); + int retval = MPI_SUCCESS; + int rank = smpi_mpi_comm_world_rank_self(); - SIMIX_mutex_lock(smpi_pending_recv_requests_mutex[rank]); - xbt_fifo_push(smpi_pending_recv_requests[rank], request); - SIMIX_mutex_unlock(smpi_pending_recv_requests_mutex[rank]); + if (NULL != request) { + SIMIX_mutex_lock(smpi_global->pending_recv_request_queues_mutexes[rank]); + xbt_fifo_push(smpi_global->pending_recv_request_queues[rank], request); + SIMIX_mutex_unlock(smpi_global->pending_recv_request_queues_mutexes[rank]); + } - if (MSG_process_is_suspended(smpi_receiver_processes[rank])) { - MSG_process_resume(smpi_receiver_processes[rank]); + if (SIMIX_process_is_suspended(smpi_global->receiver_processes[rank])) { + SIMIX_process_resume(smpi_global->receiver_processes[rank]); } + + return retval; } void smpi_wait(smpi_mpi_request_t *request, smpi_mpi_status_t *status) { - smx_process_t self; + smx_process_t self = SIMIX_process_self(); int suspend = 0; - self = SIMIX_process_self(); if (NULL != request) { SIMIX_mutex_lock(request->mutex); if (!request->completed) { - xbt_fifo_push(request->waitlist, self); - suspend = 1; - } - SIMIX_mutex_unlock(request->mutex); - if (suspend) { - SIMIX_suspend(self); + SIMIX_cond_wait(request->cond, request->mutex); } - if (NULL != status && MPI_STATUS_IGNORE != status) { - SIMIX_mutex_lock(request->mutex); + if (NULL != status) { status->MPI_SOURCE = request->src; - SIMIX_mutex_unlock(request->mutex); } + SIMIX_mutex_unlock(request->mutex); } } @@ -663,14 +793,18 @@ unsigned int smpi_sleep(unsigned int seconds) sleep_action = SIMIX_action_sleep(host, seconds); mutex = SIMIX_mutex_init(); cond = SIMIX_cond_init(); - SIMIX_mutex_lock(mutex); + SIMIX_register_condition_to_action(sleep_action, cond); SIMIX_register_action_to_condition(sleep_action, cond); + SIMIX_mutex_lock(mutex); SIMIX_cond_wait(cond, mutex); SIMIX_mutex_unlock(mutex); + SIMIX_mutex_destroy(mutex); SIMIX_cond_destroy(cond); + // FIXME: check for success/failure? + smpi_bench_begin(); return 0; } @@ -678,9 +812,9 @@ unsigned int smpi_sleep(unsigned int seconds) void smpi_exit(int status) { smpi_bench_end(); - SIMIX_mutex_lock(smpi_running_hosts_mutex); - smpi_running_hosts--; - SIMIX_mutex_unlock(smpi_running_hosts_mutex); + SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex); + smpi_global->running_hosts_count--; + SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex); SIMIX_process_kill(SIMIX_process_self()); return; }