Logo AND Algorithmique Numérique Distribuée

Public GIT Repository
Fix windows build (don't ask why this helps)
[simgrid.git] / src / smpi / smpi_base.c
1 #include <stdio.h>
2 #include <signal.h>
3 #include <sys/time.h>
4
5 #include "private.h"
6
7 SMPI_Global_t     smpi_global     = NULL;
8
9 SMPI_MPI_Global_t smpi_mpi_global = NULL;
10
11 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi,XBT_LOG_ROOT_CAT, "All SMPI categories (see \ref SMPI_API)");
12
13 int inline smpi_mpi_comm_size(smpi_mpi_communicator_t *comm)
14 {
15         return comm->size;
16 }
17
18 // FIXME: smarter algorithm?
19 int smpi_mpi_comm_rank(smpi_mpi_communicator_t *comm, smx_host_t host)
20 {
21         int i;
22
23         for(i = comm->size - 1; i > 0 && host != comm->hosts[i]; i--);
24
25         return i;
26 }
27
28 int inline smpi_mpi_comm_rank_self(smpi_mpi_communicator_t *comm)
29 {
30         return smpi_mpi_comm_rank(comm, SIMIX_host_self());
31 }
32
33 //int smpi_mpi_comm_world_rank_self()
34 //{
35 //      return smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, SIMIX_host_self());
36 //}
37
38 int smpi_sender(int argc, char **argv)
39 {
40         smx_process_t self;
41         smx_host_t shost;
42         int rank;
43
44         xbt_fifo_t request_queue;
45         smx_mutex_t request_queue_mutex;
46         int size;
47
48         int running_hosts_count;
49
50         smpi_mpi_request_t *request;
51
52         smx_host_t dhost;
53
54         smx_action_t communicate_action;
55
56         smpi_received_message_t *message;
57
58         int drank;
59
60         smx_process_t receiver_process;
61
62         self  = SIMIX_process_self();
63         shost = SIMIX_host_self();
64         rank  = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, shost);
65
66         // make sure root is done before own initialization
67         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
68         if (!smpi_global->root_ready) {
69                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
70         }
71         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
72
73         request_queue       = smpi_global->pending_send_request_queues[rank];
74         request_queue_mutex = smpi_global->pending_send_request_queues_mutexes[rank];
75         size                = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world);
76
77         smpi_global->sender_processes[rank] = self;
78
79         // wait for all nodes to signal initializatin complete
80         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
81         smpi_global->ready_process_count++;
82         if (smpi_global->ready_process_count < 3 * size) {
83                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
84         } else {
85                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
86         }
87         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
88
89         do {
90
91                 SIMIX_mutex_lock(request_queue_mutex);
92                 request = xbt_fifo_shift(request_queue);
93                 SIMIX_mutex_unlock(request_queue_mutex);
94
95                 if (NULL == request) {
96                         SIMIX_process_suspend(self);
97                 } else {
98
99                         SIMIX_mutex_lock(request->mutex);
100
101                         // copy request to appropriate received queue
102                         message       = xbt_mallocator_get(smpi_global->message_mallocator);
103                         message->comm = request->comm;
104                         message->src  = request->src;
105                         message->dst  = request->dst;
106                         message->tag  = request->tag;
107                         message->buf  = xbt_malloc(request->datatype->size * request->count);
108                         memcpy(message->buf, request->buf, request->datatype->size * request->count);
109
110                         dhost = request->comm->hosts[request->dst];
111                         drank = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, dhost);
112
113                         SIMIX_mutex_lock(smpi_global->received_message_queues_mutexes[drank]);
114                         xbt_fifo_push(smpi_global->received_message_queues[drank], message);
115                         SIMIX_mutex_unlock(smpi_global->received_message_queues_mutexes[drank]);
116
117                         request->completed = 1;
118
119                         communicate_action = SIMIX_action_communicate(shost, dhost,
120                                 NULL, request->datatype->size * request->count * 1.0, -1.0);
121
122                         SIMIX_register_condition_to_action(communicate_action, request->cond);
123                         SIMIX_register_action_to_condition(communicate_action, request->cond);
124
125                         SIMIX_cond_wait(request->cond, request->mutex);
126
127                         SIMIX_mutex_unlock(request->mutex);
128
129                         // wake up receiver if necessary
130                         receiver_process = smpi_global->receiver_processes[drank];
131
132                         if (SIMIX_process_is_suspended(receiver_process)) {
133                                 SIMIX_process_resume(receiver_process);
134                         }
135
136                 }
137
138                 SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
139                 running_hosts_count = smpi_global->running_hosts_count;
140                 SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
141
142         } while (0 < running_hosts_count);
143
144         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
145         smpi_global->ready_process_count--;
146         if (smpi_global->ready_process_count == 0) {
147                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
148         } else if (smpi_global->ready_process_count < 0) {
149                 // FIXME: can't happen! abort!
150         }
151         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
152
153         return 0;
154 }
155
156 int smpi_receiver(int argc, char **argv)
157 {
158         smx_process_t self;
159         int rank;
160
161         xbt_fifo_t request_queue;
162         smx_mutex_t request_queue_mutex;
163         xbt_fifo_t message_queue;
164         smx_mutex_t message_queue_mutex;
165         int size;
166
167         int running_hosts_count;
168
169         smpi_mpi_request_t *request;
170         smpi_received_message_t *message;
171
172         xbt_fifo_item_t request_item;
173         xbt_fifo_item_t message_item;
174
175         self  = SIMIX_process_self();
176         rank  = smpi_mpi_comm_rank_self(smpi_mpi_global->mpi_comm_world);
177
178         // make sure root is done before own initialization
179         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
180         if (!smpi_global->root_ready) {
181                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
182         }
183         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
184
185         request_queue       = smpi_global->pending_recv_request_queues[rank];
186         request_queue_mutex = smpi_global->pending_recv_request_queues_mutexes[rank];
187         message_queue       = smpi_global->received_message_queues[rank];
188         message_queue_mutex = smpi_global->received_message_queues_mutexes[rank];
189         size                = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world);
190
191         smpi_global->receiver_processes[rank] = self;
192
193         // wait for all nodes to signal initializatin complete
194         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
195         smpi_global->ready_process_count++;
196         if (smpi_global->ready_process_count < 3 * size) {
197                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
198         } else {
199                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
200         }
201         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
202
203         do {
204                 request = NULL;
205                 message = NULL;
206
207                 // FIXME: better algorithm, maybe some kind of balanced tree? or a heap?
208
209                 // FIXME: not the best way to request multiple locks...
210                 SIMIX_mutex_lock(request_queue_mutex);
211                 SIMIX_mutex_lock(message_queue_mutex);
212                 for (request_item = xbt_fifo_get_first_item(request_queue);
213                         NULL != request_item;
214                         request_item = xbt_fifo_get_next_item(request_item)) {
215                         request = xbt_fifo_get_item_content(request_item);
216                         for (message_item = xbt_fifo_get_first_item(message_queue);
217                                 NULL != message_item;
218                                 message_item = xbt_fifo_get_next_item(message_item)) {
219                                 message = xbt_fifo_get_item_content(message_item);
220                                 if (request->comm == message->comm &&
221                                                 (MPI_ANY_SOURCE == request->src || request->src == message->src) &&
222                                                 request->tag == message->tag) {
223                                         xbt_fifo_remove_item(request_queue, request_item);
224                                         xbt_fifo_remove_item(message_queue, message_item);
225                                         goto stopsearch;
226                                 }
227                         }
228                 }
229 stopsearch:
230                 SIMIX_mutex_unlock(message_queue_mutex);
231                 SIMIX_mutex_unlock(request_queue_mutex);
232
233                 if (NULL == request || NULL == message) {
234                         SIMIX_process_suspend(self);
235                 } else {
236                         SIMIX_mutex_lock(request->mutex);
237
238                         memcpy(request->buf, message->buf, request->datatype->size * request->count);
239                         request->src = message->src;
240                         request->completed = 1;
241                         SIMIX_cond_broadcast(request->cond);
242
243                         SIMIX_mutex_unlock(request->mutex);
244
245                         xbt_free(message->buf);
246                         xbt_mallocator_release(smpi_global->message_mallocator, message);
247                 }
248
249                 SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
250                 running_hosts_count = smpi_global->running_hosts_count;
251                 SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
252
253         } while (0 < running_hosts_count);
254
255         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
256         smpi_global->ready_process_count--;
257         if (smpi_global->ready_process_count == 0) {
258                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
259         } else if (smpi_global->ready_process_count < 0) {
260                 // FIXME: can't happen, abort!
261         }
262         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
263
264         return 0;
265 }
266
267 void *smpi_request_new()
268 {
269         smpi_mpi_request_t *request = xbt_new(smpi_mpi_request_t, 1);
270
271         request->completed = 0;
272         request->mutex     = SIMIX_mutex_init();
273         request->cond      = SIMIX_cond_init();
274
275         return request;
276 }
277
278 void smpi_request_free(void *pointer)
279 {
280
281         smpi_mpi_request_t *request = pointer;
282
283         if (NULL != request) {
284                 SIMIX_cond_destroy(request->cond);
285                 SIMIX_mutex_destroy(request->mutex);
286                 xbt_free(request);
287         }
288
289         return;
290 }
291
292 void smpi_request_reset(void *pointer)
293 {
294         return;
295 }
296
297
298 void *smpi_message_new()
299 {
300         return xbt_new(smpi_received_message_t, 1);
301 }
302
303 void smpi_message_free(void *pointer)
304 {
305         if (NULL != pointer) {
306                 xbt_free(pointer);
307         }
308
309         return;
310 }
311
312 void smpi_message_reset(void *pointer)
313 {
314         return;
315 }
316
317 void smpi_global_init()
318 {
319         int i;
320
321         int size = SIMIX_host_get_number();
322
323         smpi_global = xbt_new(s_SMPI_Global_t, 1);
324
325         // config variable
326         smpi_global->reference_speed                     = SMPI_DEFAULT_SPEED;
327
328         smpi_global->root_ready                          = 0;
329         smpi_global->ready_process_count                 = 0;
330
331         // start/stop
332         smpi_global->start_stop_mutex                    = SIMIX_mutex_init();
333         smpi_global->start_stop_cond                     = SIMIX_cond_init();
334
335         // processes
336         smpi_global->sender_processes                    = xbt_new(smx_process_t, size);
337         smpi_global->receiver_processes                  = xbt_new(smx_process_t, size);
338
339         // running hosts
340         smpi_global->running_hosts_count_mutex           = SIMIX_mutex_init();
341         smpi_global->running_hosts_count                 = 0;
342
343         // mallocators
344         smpi_global->request_mallocator                  = xbt_mallocator_new(SMPI_REQUEST_MALLOCATOR_SIZE,
345                                                              smpi_request_new, smpi_request_free, smpi_request_reset);
346         smpi_global->message_mallocator                  = xbt_mallocator_new(SMPI_MESSAGE_MALLOCATOR_SIZE,
347                                                              smpi_message_new, smpi_message_free, smpi_message_reset);
348
349         //
350         smpi_global->pending_send_request_queues         = xbt_new(xbt_fifo_t,  size);
351         smpi_global->pending_send_request_queues_mutexes = xbt_new(smx_mutex_t, size);
352         smpi_global->pending_recv_request_queues         = xbt_new(xbt_fifo_t,  size);
353         smpi_global->pending_recv_request_queues_mutexes = xbt_new(smx_mutex_t, size);
354         smpi_global->received_message_queues             = xbt_new(xbt_fifo_t,  size);
355         smpi_global->received_message_queues_mutexes     = xbt_new(smx_mutex_t, size);
356         smpi_global->timers                              = xbt_new(xbt_os_timer_t, size);
357         smpi_global->timers_mutexes                      = xbt_new(smx_mutex_t, size);
358
359         for(i = 0; i < size; i++) {
360                 smpi_global->pending_send_request_queues[i]         = xbt_fifo_new();
361                 smpi_global->pending_send_request_queues_mutexes[i] = SIMIX_mutex_init();
362                 smpi_global->pending_recv_request_queues[i]         = xbt_fifo_new();
363                 smpi_global->pending_recv_request_queues_mutexes[i] = SIMIX_mutex_init();
364                 smpi_global->received_message_queues[i]             = xbt_fifo_new();
365                 smpi_global->received_message_queues_mutexes[i]     = SIMIX_mutex_init();
366                 smpi_global->timers[i]                              = xbt_os_timer_new();
367                 smpi_global->timers_mutexes[i]                      = SIMIX_mutex_init();
368         }
369
370 }
371
372 void smpi_global_destroy()
373 {
374         int i;
375
376         int size = SIMIX_host_get_number();
377
378         // start/stop
379         SIMIX_mutex_destroy(smpi_global->start_stop_mutex);
380         SIMIX_cond_destroy(smpi_global->start_stop_cond);
381
382         // processes
383         xbt_free(smpi_global->sender_processes);
384         xbt_free(smpi_global->receiver_processes);
385
386         // running hosts
387         SIMIX_mutex_destroy(smpi_global->running_hosts_count_mutex);
388
389         // mallocators
390         xbt_mallocator_free(smpi_global->request_mallocator);
391         xbt_mallocator_free(smpi_global->message_mallocator);
392
393         for(i = 0; i < size; i++) {
394                 xbt_fifo_free(smpi_global->pending_send_request_queues[i]);
395                 SIMIX_mutex_destroy(smpi_global->pending_send_request_queues_mutexes[i]);
396                 xbt_fifo_free(smpi_global->pending_recv_request_queues[i]);
397                 SIMIX_mutex_destroy(smpi_global->pending_recv_request_queues_mutexes[i]);
398                 xbt_fifo_free(smpi_global->received_message_queues[i]);
399                 SIMIX_mutex_destroy(smpi_global->received_message_queues_mutexes[i]);
400                 xbt_os_timer_free(smpi_global->timers[i]);
401                 SIMIX_mutex_destroy(smpi_global->timers_mutexes[i]);
402         }
403
404         xbt_free(smpi_global->pending_send_request_queues);
405         xbt_free(smpi_global->pending_send_request_queues_mutexes);
406         xbt_free(smpi_global->pending_recv_request_queues);
407         xbt_free(smpi_global->pending_recv_request_queues_mutexes);
408         xbt_free(smpi_global->received_message_queues);
409         xbt_free(smpi_global->received_message_queues_mutexes);
410         xbt_free(smpi_global->timers);
411         xbt_free(smpi_global->timers_mutexes);
412
413         xbt_free(smpi_global);
414 }
415
416 int smpi_run_simulation(int argc, char **argv)
417 {
418         smx_cond_t   cond           = NULL;
419         smx_action_t action         = NULL;
420
421         xbt_fifo_t   actions_failed = xbt_fifo_new();
422         xbt_fifo_t   actions_done   = xbt_fifo_new();
423
424         srand(SMPI_RAND_SEED);
425
426         SIMIX_global_init(&argc, argv);
427
428         SIMIX_function_register("smpi_simulated_main", smpi_simulated_main);
429         SIMIX_function_register("smpi_sender",         smpi_sender);
430         SIMIX_function_register("smpi_receiver",       smpi_receiver);
431
432         // FIXME: ought to verify these files...
433         SIMIX_create_environment(argv[1]);
434
435         // must initialize globals between creating environment and launching app....
436         smpi_global_init();
437
438         SIMIX_launch_application(argv[2]);
439
440         /* Prepare to display some more info when dying on Ctrl-C pressing */
441         // FIXME: doesn't work
442         //signal(SIGINT, inthandler);
443
444         /* Clean IO before the run */
445         fflush(stdout);
446         fflush(stderr);
447
448         while (SIMIX_solve(actions_done, actions_failed) != -1.0) {
449                 while ((action = xbt_fifo_pop(actions_failed))) {
450                         DEBUG1("** %s failed **", action->name);
451                         while ((cond = xbt_fifo_pop(action->cond_list))) {
452                                 SIMIX_cond_broadcast(cond);
453                         }
454                         SIMIX_action_destroy(action);
455                 }
456                 while ((action = xbt_fifo_pop(actions_done))) {
457                         DEBUG1("** %s done **",action->name);
458                         while ((cond = xbt_fifo_pop(action->cond_list))) {
459                                 SIMIX_cond_broadcast(cond);
460                         }
461                         SIMIX_action_destroy(action);
462                 }
463         }
464
465         xbt_fifo_free(actions_failed);
466         xbt_fifo_free(actions_done);
467
468         INFO1("simulation time %g", SIMIX_get_clock());
469
470         smpi_global_destroy();
471
472         SIMIX_clean();
473
474         return 0;
475 }
476
477 void smpi_mpi_land_func(void *x, void *y, void *z)
478 {
479         *(int *)z = *(int *)x && *(int *)y;
480 }
481
482 void smpi_mpi_sum_func(void *x, void *y, void *z)
483 {
484         *(int *)z = *(int *)x + *(int *)y;
485 }
486
487
488 void smpi_mpi_init()
489 {
490         smx_process_t process;
491         smx_host_t host;
492         smx_host_t *hosts;
493         int size;
494
495         SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
496         smpi_global->running_hosts_count++;
497         SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
498
499         // initialize some local variables
500         process = SIMIX_process_self();
501         host    = SIMIX_host_self();
502         hosts   = SIMIX_host_get_table();
503         size    = SIMIX_host_get_number();
504
505         // node 0 sets the globals
506         if (host == hosts[0]) {
507
508                 smpi_mpi_global                                = xbt_new(s_SMPI_MPI_Global_t, 1);
509
510                 // global communicator
511                 smpi_mpi_global->mpi_comm_world                = xbt_new(smpi_mpi_communicator_t, 1);
512                 smpi_mpi_global->mpi_comm_world->size          = size;
513                 smpi_mpi_global->mpi_comm_world->barrier_count = 0;
514                 smpi_mpi_global->mpi_comm_world->barrier_mutex = SIMIX_mutex_init();
515                 smpi_mpi_global->mpi_comm_world->barrier_cond  = SIMIX_cond_init();
516                 smpi_mpi_global->mpi_comm_world->hosts         = hosts;
517                 smpi_mpi_global->mpi_comm_world->processes     = xbt_new(smx_process_t, size);
518                 smpi_mpi_global->mpi_comm_world->processes[0]  = process;
519
520                 // mpi datatypes
521                 smpi_mpi_global->mpi_byte                      = xbt_new(smpi_mpi_datatype_t, 1);
522                 smpi_mpi_global->mpi_byte->size                = (size_t)1;
523                 smpi_mpi_global->mpi_int                       = xbt_new(smpi_mpi_datatype_t, 1);
524                 smpi_mpi_global->mpi_int->size                 = sizeof(int);
525                 smpi_mpi_global->mpi_double                    = xbt_new(smpi_mpi_datatype_t, 1);
526                 smpi_mpi_global->mpi_double->size              = sizeof(double);
527
528                 // mpi operations
529                 smpi_mpi_global->mpi_land                      = xbt_new(smpi_mpi_op_t, 1);
530                 smpi_mpi_global->mpi_land->func                = smpi_mpi_land_func;
531                 smpi_mpi_global->mpi_sum                       = xbt_new(smpi_mpi_op_t, 1);
532                 smpi_mpi_global->mpi_sum->func                 = smpi_mpi_sum_func;
533
534                 // signal all nodes to perform initialization
535                 SIMIX_mutex_lock(smpi_global->start_stop_mutex);
536                 smpi_global->root_ready = 1;
537                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
538                 SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
539
540         } else {
541
542                 // make sure root is done before own initialization
543                 SIMIX_mutex_lock(smpi_global->start_stop_mutex);
544                 if (!smpi_global->root_ready) {
545                         SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
546                 }
547                 SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
548
549                 smpi_mpi_global->mpi_comm_world->processes[smpi_mpi_comm_rank_self(smpi_mpi_global->mpi_comm_world)] = process;
550         }
551
552         // wait for all nodes to signal initializatin complete
553         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
554         smpi_global->ready_process_count++;
555         if (smpi_global->ready_process_count < 3 * size) {
556                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
557         } else {
558                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
559         }
560         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
561
562         return;
563 }
564
565 void smpi_mpi_finalize()
566 {
567         int i;
568
569         SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
570         i = --smpi_global->running_hosts_count;
571         SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
572
573         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
574         smpi_global->ready_process_count--;
575         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
576
577         if (0 >= i) {
578
579                 // wake up senders/receivers
580                 for (i = 0; i < smpi_mpi_global->mpi_comm_world->size; i++) {
581                         if (SIMIX_process_is_suspended(smpi_global->sender_processes[i])) {
582                                 SIMIX_process_resume(smpi_global->sender_processes[i]);
583                         }
584                         if (SIMIX_process_is_suspended(smpi_global->receiver_processes[i])) {
585                                 SIMIX_process_resume(smpi_global->receiver_processes[i]);
586                         }
587                 }
588
589                 // wait for senders/receivers to exit...
590                 SIMIX_mutex_lock(smpi_global->start_stop_mutex);
591                 if (smpi_global->ready_process_count > 0) {
592                         SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
593                 }
594                 SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
595
596                 SIMIX_mutex_destroy(smpi_mpi_global->mpi_comm_world->barrier_mutex);
597                 SIMIX_cond_destroy(smpi_mpi_global->mpi_comm_world->barrier_cond);
598                 xbt_free(smpi_mpi_global->mpi_comm_world->processes);
599                 xbt_free(smpi_mpi_global->mpi_comm_world);
600
601                 xbt_free(smpi_mpi_global->mpi_byte);
602                 xbt_free(smpi_mpi_global->mpi_int);
603                 xbt_free(smpi_mpi_global->mpi_double);
604
605                 xbt_free(smpi_mpi_global->mpi_land);
606                 xbt_free(smpi_mpi_global->mpi_sum);
607
608                 xbt_free(smpi_mpi_global);
609         }
610
611 }
612
613 // FIXME: could cause trouble with multithreaded procs on same host...
614 void smpi_bench_begin()
615 {
616         int rank = smpi_mpi_comm_rank_self(smpi_mpi_global->mpi_comm_world);
617         SIMIX_mutex_lock(smpi_global->timers_mutexes[rank]);
618         xbt_os_timer_start(smpi_global->timers[rank]);
619         return;
620 }
621
622 void smpi_bench_end()
623 {
624         int rank = smpi_mpi_comm_rank_self(smpi_mpi_global->mpi_comm_world);
625         double duration;
626         smx_host_t host;
627         smx_action_t compute_action;
628         smx_mutex_t mutex;
629         smx_cond_t cond;
630
631         xbt_os_timer_stop(smpi_global->timers[rank]);
632
633         duration       = xbt_os_timer_elapsed(smpi_global->timers[rank]);
634         SIMIX_mutex_unlock(smpi_global->timers_mutexes[rank]);
635
636         host           = smpi_mpi_global->mpi_comm_world->hosts[rank];
637         compute_action = SIMIX_action_execute(host, NULL, duration * SMPI_DEFAULT_SPEED);
638         mutex          = SIMIX_mutex_init();
639         cond           = SIMIX_cond_init();
640
641         SIMIX_register_condition_to_action(compute_action, cond);
642         SIMIX_register_action_to_condition(compute_action, cond);
643         SIMIX_mutex_lock(mutex);
644         SIMIX_cond_wait(cond, mutex);
645         SIMIX_mutex_unlock(mutex);
646
647         SIMIX_mutex_destroy(mutex);
648         SIMIX_cond_destroy(cond);
649
650         // FIXME: check for success/failure?
651
652         return;
653 }
654
655 void smpi_barrier(smpi_mpi_communicator_t *comm)
656 {
657
658         SIMIX_mutex_lock(comm->barrier_mutex);
659         if(++comm->barrier_count < comm->size) {
660                 SIMIX_cond_wait(comm->barrier_cond, comm->barrier_mutex);
661         } else {
662                 comm->barrier_count = 0;
663                 SIMIX_cond_broadcast(comm->barrier_cond);
664         }
665         SIMIX_mutex_unlock(comm->barrier_mutex);
666
667         return;
668 }
669
670 // FIXME: smarter algorithm...
671 int smpi_comm_rank(smpi_mpi_communicator_t *comm, smx_host_t host)
672 {
673         int i;
674         for(i = 0; i < comm->size && host != comm->hosts[i]; i++);
675         if (i >= comm->size) i = -1;
676         return i;
677 }
678
679 int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype,
680         int src, int dst, int tag, smpi_mpi_communicator_t *comm, smpi_mpi_request_t **request)
681 {
682         int retval = MPI_SUCCESS;
683
684         *request = NULL;
685
686         if (0 > count) {
687                 retval = MPI_ERR_COUNT;
688         } else if (NULL == buf) {
689                 retval = MPI_ERR_INTERN;
690         } else if (NULL == datatype) {
691                 retval = MPI_ERR_TYPE;
692         } else if (NULL == comm) {
693                 retval = MPI_ERR_COMM;
694         } else if (MPI_ANY_SOURCE != src && (0 > src || comm->size <= src)) {
695                 retval = MPI_ERR_RANK;
696         } else if (0 > dst || comm->size <= dst) {
697                 retval = MPI_ERR_RANK;
698         } else if (0 > tag) {
699                 retval = MPI_ERR_TAG;
700         } else {
701                 *request = xbt_mallocator_get(smpi_global->request_mallocator);
702                 (*request)->comm       = comm;
703                 (*request)->src        = src;
704                 (*request)->dst        = dst;
705                 (*request)->tag        = tag;
706                 (*request)->buf        = buf;
707                 (*request)->count      = count;
708                 (*request)->datatype   = datatype;
709         }
710         return retval;
711 }
712
713 int smpi_isend(smpi_mpi_request_t *request)
714 {
715         int retval = MPI_SUCCESS;
716         int rank   = smpi_mpi_comm_rank_self(smpi_mpi_global->mpi_comm_world);
717
718         if (NULL != request) {
719                 SIMIX_mutex_lock(smpi_global->pending_send_request_queues_mutexes[rank]);
720                 xbt_fifo_push(smpi_global->pending_send_request_queues[rank], request);
721                 SIMIX_mutex_unlock(smpi_global->pending_send_request_queues_mutexes[rank]);
722         }
723
724         if (SIMIX_process_is_suspended(smpi_global->sender_processes[rank])) {
725                 SIMIX_process_resume(smpi_global->sender_processes[rank]);
726         }
727
728         return retval;
729 }
730
731 int smpi_irecv(smpi_mpi_request_t *request)
732 {
733         int retval = MPI_SUCCESS;
734         int rank = smpi_mpi_comm_rank_self(smpi_mpi_global->mpi_comm_world);
735
736         if (NULL != request) {
737                 SIMIX_mutex_lock(smpi_global->pending_recv_request_queues_mutexes[rank]);
738                 xbt_fifo_push(smpi_global->pending_recv_request_queues[rank], request);
739                 SIMIX_mutex_unlock(smpi_global->pending_recv_request_queues_mutexes[rank]);
740         }
741
742         if (SIMIX_process_is_suspended(smpi_global->receiver_processes[rank])) {
743                 SIMIX_process_resume(smpi_global->receiver_processes[rank]);
744         }
745
746         return retval;
747 }
748
749 void smpi_wait(smpi_mpi_request_t *request, smpi_mpi_status_t *status)
750 {
751         if (NULL != request) {
752                 SIMIX_mutex_lock(request->mutex);
753                 if (!request->completed) {
754                         SIMIX_cond_wait(request->cond, request->mutex);
755                 }
756                 if (NULL != status) {
757                         status->MPI_SOURCE = request->src;
758                 }
759                 SIMIX_mutex_unlock(request->mutex);
760         }
761 }
762
763 // FIXME: move into own file
764 int smpi_gettimeofday(struct timeval *tv, struct timezone *tz)
765 {
766         double now;
767         int retval = 0;
768         smpi_bench_end();
769         if (NULL == tv) {
770                 retval = -1;
771         } else {
772                 now = SIMIX_get_clock();
773                 tv->tv_sec  = now;
774                 tv->tv_usec = ((now - (double)tv->tv_sec) * 1000000.0);
775         }
776         smpi_bench_begin();
777         return retval;
778 }
779
780 unsigned int smpi_sleep(unsigned int seconds)
781 {
782         smx_mutex_t mutex;
783         smx_cond_t cond;
784         smx_host_t host;
785         smx_action_t sleep_action;
786
787         smpi_bench_end();
788         host         = SIMIX_host_self();
789         sleep_action = SIMIX_action_sleep(host, seconds);
790         mutex        = SIMIX_mutex_init();
791         cond         = SIMIX_cond_init();
792
793         SIMIX_register_condition_to_action(sleep_action, cond);
794         SIMIX_register_action_to_condition(sleep_action, cond);
795         SIMIX_mutex_lock(mutex);
796         SIMIX_cond_wait(cond, mutex);
797         SIMIX_mutex_unlock(mutex);
798
799         SIMIX_mutex_destroy(mutex);
800         SIMIX_cond_destroy(cond);
801
802         // FIXME: check for success/failure?
803
804         smpi_bench_begin();
805         return 0;
806 }
807
808 void smpi_exit(int status)
809 {
810         smpi_bench_end();
811         SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
812         smpi_global->running_hosts_count--;
813         SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
814         SIMIX_process_kill(SIMIX_process_self());
815         return;
816 }