Logo AND Algorithmique Numérique Distribuée

Public GIT Repository
minor changes
[simgrid.git] / src / smpi / src / smpi_base.c
1 #include <stdio.h>
2 #include <signal.h>
3 #include <sys/time.h>
4
5 #include "private.h"
6
7 SMPI_Global_t     smpi_global     = NULL;
8
9 SMPI_MPI_Global_t smpi_mpi_global = NULL;
10
11 XBT_LOG_NEW_DEFAULT_CATEGORY(smpi, "SMPI");
12
13 int inline smpi_mpi_comm_size(smpi_mpi_communicator_t *comm)
14 {
15         return comm->size;
16 }
17
18 // FIXME: smarter algorithm?
19 int smpi_mpi_comm_rank(smpi_mpi_communicator_t *comm, smx_host_t host)
20 {
21         int i;
22
23         for(i = comm->size - 1; i > 0 && host != comm->hosts[i]; i--);
24
25         return i;
26 }
27
28 int inline smpi_mpi_comm_rank_self(smpi_mpi_communicator_t *comm)
29 {
30         return smpi_mpi_comm_rank(comm, SIMIX_host_self());
31 }
32
33 int inline smpi_mpi_comm_world_rank_self()
34 {
35         return smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, SIMIX_host_self());
36 }
37
38 int smpi_sender(int argc, char **argv)
39 {
40         smx_process_t self;
41         smx_host_t shost;
42         int rank;
43
44         xbt_fifo_t request_queue;
45         smx_mutex_t request_queue_mutex;
46         int size;
47
48         int running_hosts_count;
49
50         smpi_mpi_request_t *request;
51
52         smx_host_t dhost;
53
54         smx_action_t communicate_action;
55
56         smpi_received_message_t *message;
57
58         int drank;
59
60         smx_process_t receiver_process;
61
62         self  = SIMIX_process_self();
63         shost = SIMIX_host_self();
64         rank  = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, shost);
65
66         // make sure root is done before own initialization
67         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
68         if (!smpi_global->root_ready) {
69                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
70         }
71         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
72
73         request_queue       = smpi_global->pending_send_request_queues[rank];
74         request_queue_mutex = smpi_global->pending_send_request_queues_mutexes[rank];
75         size                = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world);
76
77         smpi_global->sender_processes[rank] = self;
78
79         // wait for all nodes to signal initializatin complete
80         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
81         smpi_global->ready_process_count++;
82         if (smpi_global->ready_process_count < 3 * size) {
83                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
84         } else {
85                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
86         }
87         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
88
89         do {
90
91                 SIMIX_mutex_lock(request_queue_mutex);
92                 request = xbt_fifo_shift(request_queue);
93                 SIMIX_mutex_unlock(request_queue_mutex);
94
95                 if (NULL == request) {
96                         SIMIX_process_suspend(self);
97                 } else {
98
99                         SIMIX_mutex_lock(request->mutex);
100
101                         // copy request to appropriate received queue
102                         message       = xbt_mallocator_get(smpi_global->message_mallocator);
103                         message->comm = request->comm;
104                         message->src  = request->src;
105                         message->dst  = request->dst;
106                         message->tag  = request->tag;
107                         message->buf  = xbt_malloc(request->datatype->size * request->count);
108                         memcpy(message->buf, request->buf, request->datatype->size * request->count);
109
110                         dhost = request->comm->hosts[request->dst];
111                         drank = smpi_mpi_comm_rank(smpi_mpi_global->mpi_comm_world, dhost);
112
113                         SIMIX_mutex_lock(smpi_global->received_message_queues_mutexes[drank]);
114                         xbt_fifo_push(smpi_global->received_message_queues[drank], message);
115                         SIMIX_mutex_unlock(smpi_global->received_message_queues_mutexes[drank]);
116
117                         request->completed = 1;
118
119                         communicate_action = SIMIX_action_communicate(shost, dhost,
120                                 "communication", request->datatype->size * request->count * 1.0, -1.0);
121
122                         SIMIX_register_condition_to_action(communicate_action, request->cond);
123                         SIMIX_register_action_to_condition(communicate_action, request->cond);
124
125                         SIMIX_cond_wait(request->cond, request->mutex);
126
127                         SIMIX_mutex_unlock(request->mutex);
128
129                         // wake up receiver if necessary
130                         receiver_process = smpi_global->receiver_processes[drank];
131
132                         if (SIMIX_process_is_suspended(receiver_process)) {
133                                 SIMIX_process_resume(receiver_process);
134                         }
135
136                 }
137
138                 SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
139                 running_hosts_count = smpi_global->running_hosts_count;
140                 SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
141
142         } while (0 < running_hosts_count);
143
144         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
145         smpi_global->ready_process_count--;
146         if (smpi_global->ready_process_count == 0) {
147                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
148         } else if (smpi_global->ready_process_count < 0) {
149                 // FIXME: can't happen! abort!
150         }
151         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
152
153         return 0;
154 }
155
156 int smpi_receiver(int argc, char **argv)
157 {
158         smx_process_t self;
159         int rank;
160
161         xbt_fifo_t request_queue;
162         smx_mutex_t request_queue_mutex;
163         xbt_fifo_t message_queue;
164         smx_mutex_t message_queue_mutex;
165         int size;
166
167         int running_hosts_count;
168
169         smpi_mpi_request_t *request;
170         smpi_received_message_t *message;
171
172         xbt_fifo_item_t request_item;
173         xbt_fifo_item_t message_item;
174
175         smx_process_t waitproc;
176
177         self  = SIMIX_process_self();
178         rank  = smpi_mpi_comm_world_rank_self();
179
180         // make sure root is done before own initialization
181         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
182         if (!smpi_global->root_ready) {
183                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
184         }
185         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
186
187         request_queue       = smpi_global->pending_recv_request_queues[rank];
188         request_queue_mutex = smpi_global->pending_recv_request_queues_mutexes[rank];
189         message_queue       = smpi_global->received_message_queues[rank];
190         message_queue_mutex = smpi_global->received_message_queues_mutexes[rank];
191         size                = smpi_mpi_comm_size(smpi_mpi_global->mpi_comm_world);
192
193         smpi_global->receiver_processes[rank] = self;
194
195         // wait for all nodes to signal initializatin complete
196         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
197         smpi_global->ready_process_count++;
198         if (smpi_global->ready_process_count < 3 * size) {
199                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
200         } else {
201                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
202         }
203         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
204
205         do {
206                 request = NULL;
207                 message = NULL;
208
209                 // FIXME: better algorithm, maybe some kind of balanced tree? or a heap?
210
211                 // FIXME: not the best way to request multiple locks...
212                 SIMIX_mutex_lock(request_queue_mutex);
213                 SIMIX_mutex_lock(message_queue_mutex);
214                 for (request_item = xbt_fifo_get_first_item(request_queue);
215                         NULL != request_item;
216                         request_item = xbt_fifo_get_next_item(request_item)) {
217                         request = xbt_fifo_get_item_content(request_item);
218                         for (message_item = xbt_fifo_get_first_item(message_queue);
219                                 NULL != message_item;
220                                 message_item = xbt_fifo_get_next_item(message_item)) {
221                                 message = xbt_fifo_get_item_content(message_item);
222                                 if (request->comm == message->comm &&
223                                                 (MPI_ANY_SOURCE == request->src || request->src == message->src) &&
224                                                 request->tag == message->tag) {
225                                         xbt_fifo_remove_item(request_queue, request_item);
226                                         xbt_fifo_remove_item(message_queue, message_item);
227                                         goto stopsearch;
228                                 }
229                         }
230                 }
231 stopsearch:
232                 SIMIX_mutex_unlock(message_queue_mutex);
233                 SIMIX_mutex_unlock(request_queue_mutex);
234
235                 if (NULL == request || NULL == message) {
236                         SIMIX_process_suspend(self);
237                 } else {
238                         SIMIX_mutex_lock(request->mutex);
239
240                         memcpy(request->buf, message->buf, request->datatype->size * request->count);
241                         request->src = message->src;
242                         request->completed = 1;
243                         SIMIX_cond_broadcast(request->cond);
244
245                         SIMIX_mutex_unlock(request->mutex);
246
247                         xbt_free(message->buf);
248                         xbt_mallocator_release(smpi_global->message_mallocator, message);
249                 }
250
251                 SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
252                 running_hosts_count = smpi_global->running_hosts_count;
253                 SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
254
255         } while (0 < running_hosts_count);
256
257         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
258         smpi_global->ready_process_count--;
259         if (smpi_global->ready_process_count == 0) {
260                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
261         } else if (smpi_global->ready_process_count < 0) {
262                 // FIXME: can't happen, abort!
263         }
264         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
265
266         return 0;
267 }
268
269 void *smpi_request_new()
270 {
271         smpi_mpi_request_t *request = xbt_new(smpi_mpi_request_t, 1);
272
273         request->completed = 0;
274         request->mutex     = SIMIX_mutex_init();
275         request->cond      = SIMIX_cond_init();
276
277         return request;
278 }
279
280 void smpi_request_free(void *pointer) {
281
282         smpi_mpi_request_t *request = pointer;
283
284         if (NULL != request) {
285                 SIMIX_cond_destroy(request->cond);
286                 SIMIX_mutex_destroy(request->mutex);
287                 xbt_free(request);
288         }
289
290         return;
291 }
292
293 void smpi_request_reset(void *pointer) {
294         return;
295 }
296
297
298 void *smpi_message_new()
299 {
300         return xbt_new(smpi_received_message_t, 1);
301 }
302
303 void smpi_message_free(void *pointer)
304 {
305         if (NULL != pointer) {
306                 xbt_free(pointer);
307         }
308
309         return;
310 }
311
312 void smpi_message_reset(void *pointer)
313 {
314         return;
315 }
316
317 void smpi_global_init()
318 {
319         int i;
320
321         int size = SIMIX_host_get_number();
322
323         smpi_global = xbt_new(s_SMPI_Global_t, 1);
324
325         // config variable
326         smpi_global->reference_speed                     = SMPI_DEFAULT_SPEED;
327
328         smpi_global->root_ready                          = 0;
329         smpi_global->ready_process_count                 = 0;
330
331         // start/stop
332         smpi_global->start_stop_mutex                    = SIMIX_mutex_init();
333         smpi_global->start_stop_cond                     = SIMIX_cond_init();
334
335         // processes
336         smpi_global->sender_processes                    = xbt_new(smx_process_t, size);
337         smpi_global->receiver_processes                  = xbt_new(smx_process_t, size);
338
339         // running hosts
340         smpi_global->running_hosts_count_mutex           = SIMIX_mutex_init();
341         smpi_global->running_hosts_count                 = 0;
342
343         // mallocators
344         smpi_global->request_mallocator                  = xbt_mallocator_new(SMPI_REQUEST_MALLOCATOR_SIZE,
345                                                              smpi_request_new, smpi_request_free, smpi_request_reset);
346         smpi_global->message_mallocator                  = xbt_mallocator_new(SMPI_MESSAGE_MALLOCATOR_SIZE,
347                                                              smpi_message_new, smpi_message_free, smpi_message_reset);
348
349         //
350         smpi_global->pending_send_request_queues         = xbt_new(xbt_fifo_t,  size);
351         smpi_global->pending_send_request_queues_mutexes = xbt_new(smx_mutex_t, size);
352         smpi_global->pending_recv_request_queues         = xbt_new(xbt_fifo_t,  size);
353         smpi_global->pending_recv_request_queues_mutexes = xbt_new(smx_mutex_t, size);
354         smpi_global->received_message_queues             = xbt_new(xbt_fifo_t,  size);
355         smpi_global->received_message_queues_mutexes     = xbt_new(smx_mutex_t, size);
356         smpi_global->timers                              = xbt_new(xbt_os_timer_t, size);
357         smpi_global->timers_mutexes                      = xbt_new(smx_mutex_t, size);
358
359         for(i = 0; i < size; i++) {
360                 smpi_global->pending_send_request_queues[i]         = xbt_fifo_new();
361                 smpi_global->pending_send_request_queues_mutexes[i] = SIMIX_mutex_init();
362                 smpi_global->pending_recv_request_queues[i]         = xbt_fifo_new();
363                 smpi_global->pending_recv_request_queues_mutexes[i] = SIMIX_mutex_init();
364                 smpi_global->received_message_queues[i]             = xbt_fifo_new();
365                 smpi_global->received_message_queues_mutexes[i]     = SIMIX_mutex_init();
366                 smpi_global->timers[i]                              = xbt_os_timer_new();
367                 smpi_global->timers_mutexes[i]                      = SIMIX_mutex_init();
368         }
369
370 }
371
372 void smpi_global_destroy()
373 {
374         int i;
375
376         int size = SIMIX_host_get_number();
377
378         // start/stop
379         SIMIX_mutex_destroy(smpi_global->start_stop_mutex);
380         SIMIX_cond_destroy(smpi_global->start_stop_cond);
381
382         // processes
383         xbt_free(smpi_global->sender_processes);
384         xbt_free(smpi_global->receiver_processes);
385
386         // running hosts
387         SIMIX_mutex_destroy(smpi_global->running_hosts_count_mutex);
388
389         // mallocators
390         xbt_mallocator_free(smpi_global->request_mallocator);
391         xbt_mallocator_free(smpi_global->message_mallocator);
392
393         for(i = 0; i < size; i++) {
394                 xbt_fifo_free(smpi_global->pending_send_request_queues[i]);
395                 SIMIX_mutex_destroy(smpi_global->pending_send_request_queues_mutexes[i]);
396                 xbt_fifo_free(smpi_global->pending_recv_request_queues[i]);
397                 SIMIX_mutex_destroy(smpi_global->pending_recv_request_queues_mutexes[i]);
398                 xbt_fifo_free(smpi_global->received_message_queues[i]);
399                 SIMIX_mutex_destroy(smpi_global->received_message_queues_mutexes[i]);
400                 xbt_os_timer_free(smpi_global->timers[i]);
401                 SIMIX_mutex_destroy(smpi_global->timers_mutexes[i]);
402         }
403
404         xbt_free(smpi_global->pending_send_request_queues);
405         xbt_free(smpi_global->pending_send_request_queues_mutexes);
406         xbt_free(smpi_global->pending_recv_request_queues);
407         xbt_free(smpi_global->pending_recv_request_queues_mutexes);
408         xbt_free(smpi_global->received_message_queues);
409         xbt_free(smpi_global->received_message_queues_mutexes);
410         xbt_free(smpi_global->timers);
411         xbt_free(smpi_global->timers_mutexes);
412
413         xbt_free(smpi_global);
414 }
415
416 int smpi_run_simulation(int argc, char **argv)
417 {
418         smx_cond_t   cond           = NULL;
419         smx_action_t action         = NULL;
420
421         xbt_fifo_t   actions_failed = xbt_fifo_new();
422         xbt_fifo_t   actions_done   = xbt_fifo_new();
423
424         srand(SMPI_RAND_SEED);
425
426         SIMIX_global_init(&argc, argv);
427
428         SIMIX_function_register("smpi_simulated_main", smpi_simulated_main);
429         SIMIX_function_register("smpi_sender",         smpi_sender);
430         SIMIX_function_register("smpi_receiver",       smpi_receiver);
431
432         // FIXME: ought to verify these files...
433         SIMIX_create_environment(argv[1]);
434
435         // must initialize globals between creating environment and launching app....
436         smpi_global_init();
437
438         SIMIX_launch_application(argv[2]);
439
440         /* Prepare to display some more info when dying on Ctrl-C pressing */
441         // FIXME: doesn't work
442         //signal(SIGINT, inthandler);
443
444         /* Clean IO before the run */
445         fflush(stdout);
446         fflush(stderr);
447
448         while (SIMIX_solve(actions_done, actions_failed) != -1.0) {
449                 while (action = xbt_fifo_pop(actions_failed)) {
450                         DEBUG1("** %s failed **", action->name);
451                         while (cond = xbt_fifo_pop(action->cond_list)) {
452                                 SIMIX_cond_broadcast(cond);
453                         }
454                         SIMIX_action_destroy(action);
455                 }
456                 while (action = xbt_fifo_pop(actions_done)) {
457                         DEBUG1("** %s done **",action->name);
458                         while (cond = xbt_fifo_pop(action->cond_list)) {
459                                 SIMIX_cond_broadcast(cond);
460                         }
461                         SIMIX_action_destroy(action);
462                 }
463         }
464
465         xbt_fifo_free(actions_failed);
466         xbt_fifo_free(actions_done);
467
468         INFO1("simulation time %g", SIMIX_get_clock());
469
470         smpi_global_destroy();
471
472         SIMIX_clean();
473
474         return 0;
475 }
476
477 void smpi_mpi_land_func(void *x, void *y, void *z)
478 {
479         *(int *)z = *(int *)x && *(int *)y;
480 }
481
482 void smpi_mpi_sum_func(void *x, void *y, void *z)
483 {
484         *(int *)z = *(int *)x + *(int *)y;
485 }
486
487
488 void smpi_mpi_init()
489 {
490         int i;
491         smx_process_t process;
492         smx_host_t host;
493         smx_host_t *hosts;
494         int size;
495         double duration;
496
497         SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
498         smpi_global->running_hosts_count++;
499         SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
500
501         // initialize some local variables
502         process = SIMIX_process_self();
503         host    = SIMIX_host_self();
504         hosts   = SIMIX_host_get_table();
505         size    = SIMIX_host_get_number();
506
507         // node 0 sets the globals
508         if (host == hosts[0]) {
509
510                 smpi_mpi_global                                = xbt_new(s_SMPI_MPI_Global_t, 1);
511
512                 // global communicator
513                 smpi_mpi_global->mpi_comm_world                = xbt_new(smpi_mpi_communicator_t, 1);
514                 smpi_mpi_global->mpi_comm_world->size          = size;
515                 smpi_mpi_global->mpi_comm_world->barrier_count = 0;
516                 smpi_mpi_global->mpi_comm_world->barrier_mutex = SIMIX_mutex_init();
517                 smpi_mpi_global->mpi_comm_world->barrier_cond  = SIMIX_cond_init();
518                 smpi_mpi_global->mpi_comm_world->hosts         = hosts;
519                 smpi_mpi_global->mpi_comm_world->processes     = xbt_new(smx_process_t, size);
520                 smpi_mpi_global->mpi_comm_world->processes[0]  = process;
521
522                 // mpi datatypes
523                 smpi_mpi_global->mpi_byte                      = xbt_new(smpi_mpi_datatype_t, 1);
524                 smpi_mpi_global->mpi_byte->size                = (size_t)1;
525                 smpi_mpi_global->mpi_int                       = xbt_new(smpi_mpi_datatype_t, 1);
526                 smpi_mpi_global->mpi_int->size                 = sizeof(int);
527                 smpi_mpi_global->mpi_double                    = xbt_new(smpi_mpi_datatype_t, 1);
528                 smpi_mpi_global->mpi_double->size              = sizeof(double);
529
530                 // mpi operations
531                 smpi_mpi_global->mpi_land                      = xbt_new(smpi_mpi_op_t, 1);
532                 smpi_mpi_global->mpi_land->func                = smpi_mpi_land_func;
533                 smpi_mpi_global->mpi_sum                       = xbt_new(smpi_mpi_op_t, 1);
534                 smpi_mpi_global->mpi_sum->func                 = smpi_mpi_sum_func;
535
536                 // signal all nodes to perform initialization
537                 SIMIX_mutex_lock(smpi_global->start_stop_mutex);
538                 smpi_global->root_ready = 1;
539                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
540                 SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
541
542         } else {
543
544                 // make sure root is done before own initialization
545                 SIMIX_mutex_lock(smpi_global->start_stop_mutex);
546                 if (!smpi_global->root_ready) {
547                         SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
548                 }
549                 SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
550
551                 smpi_mpi_global->mpi_comm_world->processes[smpi_mpi_comm_world_rank_self()] = process;
552         }
553
554         // wait for all nodes to signal initializatin complete
555         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
556         smpi_global->ready_process_count++;
557         if (smpi_global->ready_process_count < 3 * size) {
558                 SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
559         } else {
560                 SIMIX_cond_broadcast(smpi_global->start_stop_cond);
561         }
562         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
563
564         return;
565 }
566
567 void smpi_mpi_finalize()
568 {
569         int i;
570
571         SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
572         i = --smpi_global->running_hosts_count;
573         SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
574
575         SIMIX_mutex_lock(smpi_global->start_stop_mutex);
576         smpi_global->ready_process_count--;
577         SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
578
579         if (0 >= i) {
580
581                 // wake up senders/receivers
582                 for (i = 0; i < smpi_mpi_global->mpi_comm_world->size; i++) {
583                         if (SIMIX_process_is_suspended(smpi_global->sender_processes[i])) {
584                                 SIMIX_process_resume(smpi_global->sender_processes[i]);
585                         }
586                         if (SIMIX_process_is_suspended(smpi_global->receiver_processes[i])) {
587                                 SIMIX_process_resume(smpi_global->receiver_processes[i]);
588                         }
589                 }
590
591                 // wait for senders/receivers to exit...
592                 SIMIX_mutex_lock(smpi_global->start_stop_mutex);
593                 if (smpi_global->ready_process_count > 0) {
594                         SIMIX_cond_wait(smpi_global->start_stop_cond, smpi_global->start_stop_mutex);
595                 }
596                 SIMIX_mutex_unlock(smpi_global->start_stop_mutex);
597
598                 SIMIX_mutex_destroy(smpi_mpi_global->mpi_comm_world->barrier_mutex);
599                 SIMIX_cond_destroy(smpi_mpi_global->mpi_comm_world->barrier_cond);
600                 xbt_free(smpi_mpi_global->mpi_comm_world->processes);
601                 xbt_free(smpi_mpi_global->mpi_comm_world);
602
603                 xbt_free(smpi_mpi_global->mpi_byte);
604                 xbt_free(smpi_mpi_global->mpi_int);
605                 xbt_free(smpi_mpi_global->mpi_double);
606
607                 xbt_free(smpi_mpi_global->mpi_land);
608                 xbt_free(smpi_mpi_global->mpi_sum);
609
610                 xbt_free(smpi_mpi_global);
611         }
612
613 }
614
615 // FIXME: could cause trouble with multithreaded procs on same host...
616 void smpi_bench_begin()
617 {
618         int rank = smpi_mpi_comm_world_rank_self();
619         SIMIX_mutex_lock(smpi_global->timers_mutexes[rank]);
620         xbt_os_timer_start(smpi_global->timers[rank]);
621         return;
622 }
623
624 void smpi_bench_end()
625 {
626         int rank = smpi_mpi_comm_world_rank_self();
627         double duration;
628         smx_host_t host;
629         smx_action_t compute_action;
630         smx_mutex_t mutex;
631         smx_cond_t cond;
632
633         xbt_os_timer_stop(smpi_global->timers[rank]);
634
635         duration       = xbt_os_timer_elapsed(smpi_global->timers[rank]);
636         SIMIX_mutex_unlock(smpi_global->timers_mutexes[rank]);
637
638         host           = smpi_mpi_global->mpi_comm_world->hosts[rank];
639         compute_action = SIMIX_action_execute(host, "computation", duration * SMPI_DEFAULT_SPEED);
640         mutex          = SIMIX_mutex_init();
641         cond           = SIMIX_cond_init();
642
643         SIMIX_register_condition_to_action(compute_action, cond);
644         SIMIX_register_action_to_condition(compute_action, cond);
645         SIMIX_mutex_lock(mutex);
646         SIMIX_cond_wait(cond, mutex);
647         SIMIX_mutex_unlock(mutex);
648
649         SIMIX_mutex_destroy(mutex);
650         SIMIX_cond_destroy(cond);
651
652         // FIXME: check for success/failure?
653
654         return;
655 }
656
657 void smpi_barrier(smpi_mpi_communicator_t *comm) {
658
659         SIMIX_mutex_lock(comm->barrier_mutex);
660         if(++comm->barrier_count < comm->size) {
661                 SIMIX_cond_wait(comm->barrier_cond, comm->barrier_mutex);
662         } else {
663                 comm->barrier_count = 0;
664                 SIMIX_cond_broadcast(comm->barrier_cond);
665         }
666         SIMIX_mutex_unlock(comm->barrier_mutex);
667
668         return;
669 }
670
671 // FIXME: smarter algorithm...
672 int smpi_comm_rank(smpi_mpi_communicator_t *comm, smx_host_t host)
673 {
674         int i;
675         for(i = 0; i < comm->size && host != comm->hosts[i]; i++);
676         if (i >= comm->size) i = -1;
677         return i;
678 }
679
680 int smpi_create_request(void *buf, int count, smpi_mpi_datatype_t *datatype,
681         int src, int dst, int tag, smpi_mpi_communicator_t *comm, smpi_mpi_request_t **request)
682 {
683         int retval = MPI_SUCCESS;
684
685         *request = NULL;
686
687         if (0 > count) {
688                 retval = MPI_ERR_COUNT;
689         } else if (NULL == buf) {
690                 retval = MPI_ERR_INTERN;
691         } else if (NULL == datatype) {
692                 retval = MPI_ERR_TYPE;
693         } else if (NULL == comm) {
694                 retval = MPI_ERR_COMM;
695         } else if (MPI_ANY_SOURCE != src && (0 > src || comm->size <= src)) {
696                 retval = MPI_ERR_RANK;
697         } else if (0 > dst || comm->size <= dst) {
698                 retval = MPI_ERR_RANK;
699         } else if (0 > tag) {
700                 retval = MPI_ERR_TAG;
701         } else {
702                 *request = xbt_mallocator_get(smpi_global->request_mallocator);
703                 (*request)->comm       = comm;
704                 (*request)->src        = src;
705                 (*request)->dst        = dst;
706                 (*request)->tag        = tag;
707                 (*request)->buf        = buf;
708                 (*request)->count      = count;
709                 (*request)->datatype   = datatype;
710         }
711         return retval;
712 }
713
714 int smpi_isend(smpi_mpi_request_t *request)
715 {
716         int retval = MPI_SUCCESS;
717         int rank   = smpi_mpi_comm_world_rank_self();
718
719         if (NULL != request) {
720                 SIMIX_mutex_lock(smpi_global->pending_send_request_queues_mutexes[rank]);
721                 xbt_fifo_push(smpi_global->pending_send_request_queues[rank], request);
722                 SIMIX_mutex_unlock(smpi_global->pending_send_request_queues_mutexes[rank]);
723         }
724
725         if (SIMIX_process_is_suspended(smpi_global->sender_processes[rank])) {
726                 SIMIX_process_resume(smpi_global->sender_processes[rank]);
727         }
728
729         return retval;
730 }
731
732 int smpi_irecv(smpi_mpi_request_t *request)
733 {
734         int retval = MPI_SUCCESS;
735         int rank = smpi_mpi_comm_world_rank_self();
736
737         if (NULL != request) {
738                 SIMIX_mutex_lock(smpi_global->pending_recv_request_queues_mutexes[rank]);
739                 xbt_fifo_push(smpi_global->pending_recv_request_queues[rank], request);
740                 SIMIX_mutex_unlock(smpi_global->pending_recv_request_queues_mutexes[rank]);
741         }
742
743         if (SIMIX_process_is_suspended(smpi_global->receiver_processes[rank])) {
744                 SIMIX_process_resume(smpi_global->receiver_processes[rank]);
745         }
746
747         return retval;
748 }
749
750 void smpi_wait(smpi_mpi_request_t *request, smpi_mpi_status_t *status)
751 {
752         smx_process_t self = SIMIX_process_self();
753         int suspend = 0;
754
755         if (NULL != request) {
756                 SIMIX_mutex_lock(request->mutex);
757                 if (!request->completed) {
758                         SIMIX_cond_wait(request->cond, request->mutex);
759                 }
760                 if (NULL != status) {
761                         status->MPI_SOURCE = request->src;
762                 }
763                 SIMIX_mutex_unlock(request->mutex);
764         }
765 }
766
767 // FIXME: move into own file
768 int smpi_gettimeofday(struct timeval *tv, struct timezone *tz)
769 {
770         double now;
771         int retval = 0;
772         smpi_bench_end();
773         if (NULL == tv) {
774                 retval = -1;
775         } else {
776                 now = SIMIX_get_clock();
777                 tv->tv_sec  = now;
778                 tv->tv_usec = ((now - (double)tv->tv_sec) * 1000000.0);
779         }
780         smpi_bench_begin();
781         return retval;
782 }
783
784 unsigned int smpi_sleep(unsigned int seconds)
785 {
786         smx_mutex_t mutex;
787         smx_cond_t cond;
788         smx_host_t host;
789         smx_action_t sleep_action;
790
791         smpi_bench_end();
792         host         = SIMIX_host_self();
793         sleep_action = SIMIX_action_sleep(host, seconds);
794         mutex        = SIMIX_mutex_init();
795         cond         = SIMIX_cond_init();
796
797         SIMIX_register_condition_to_action(sleep_action, cond);
798         SIMIX_register_action_to_condition(sleep_action, cond);
799         SIMIX_mutex_lock(mutex);
800         SIMIX_cond_wait(cond, mutex);
801         SIMIX_mutex_unlock(mutex);
802
803         SIMIX_mutex_destroy(mutex);
804         SIMIX_cond_destroy(cond);
805
806         // FIXME: check for success/failure?
807
808         smpi_bench_begin();
809         return 0;
810 }
811
812 void smpi_exit(int status)
813 {
814         smpi_bench_end();
815         SIMIX_mutex_lock(smpi_global->running_hosts_count_mutex);
816         smpi_global->running_hosts_count--;
817         SIMIX_mutex_unlock(smpi_global->running_hosts_count_mutex);
818         SIMIX_process_kill(SIMIX_process_self());
819         return;
820 }