Logo AND Algorithmique Numérique Distribuée

Public GIT Repository
Fix compilation warning for unused variables
[simgrid.git] / src / surf / storage.c
1 /* Copyright (c) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012. The SimGrid Team.
2  * All rights reserved.                                                                 */
3
4 /* This program is free software; you can redistribute it and/or modify it
5  * under the terms of the license (GNU LGPL) which comes with this package.             */
6
7 #include "xbt/ex.h"
8 #include "xbt/dict.h"
9 #include "xbt/file_stat.h"
10 #include "portable.h"
11 #include "surf_private.h"
12 #include "storage_private.h"
13 #include "surf/surf_resource.h"
14
15 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(surf_storage, surf,
16                                 "Logging specific to the SURF storage module");
17
18 xbt_lib_t storage_lib;
19 int ROUTING_STORAGE_LEVEL;      //Routing for storagelevel
20 int ROUTING_STORAGE_HOST_LEVEL;
21 int SURF_STORAGE_LEVEL;
22 xbt_lib_t storage_type_lib;
23 int ROUTING_STORAGE_TYPE_LEVEL; //Routing for storage_type level
24
25 xbt_dynar_t mount_list = NULL;  /* temporary store of current mount storage */
26
27 surf_model_t surf_storage_model = NULL;
28 lmm_system_t storage_maxmin_system = NULL;
29 static int storage_selective_update = 0;
30 static xbt_swag_t
31     storage_running_action_set_that_does_not_need_being_checked = NULL;
32
33 #define GENERIC_LMM_ACTION(action) action->generic_lmm_action
34 #define GENERIC_ACTION(action) GENERIC_LMM_ACTION(action).generic_action
35
36 typedef struct surf_storage {
37   s_surf_resource_t generic_resource;
38   const char* type;
39   const char* content; /*should be a dict */
40 } s_surf_storage_t, *surf_storage_t;
41
42 static xbt_dict_t parse_storage_content(char *filename, unsigned long *used_size);
43 static void storage_action_state_set(surf_action_t action, e_surf_action_state_t state);
44 static surf_action_t storage_action_execute (void *storage, size_t size, e_surf_action_storage_type_t type);
45
46 static surf_action_t storage_action_stat(void *storage, surf_file_t stream)
47 {
48   surf_action_t action = storage_action_execute(storage,0, STAT);
49   action->file = stream;
50   action->stat = stream->content->stat;
51   return action;
52 }
53
54 static surf_action_t storage_action_open(void *storage, const char* mount, const char* path, const char* mode)
55 {
56   XBT_DEBUG("\tOpen file '%s'",path);
57   xbt_dict_t content_dict = ((storage_t)storage)->content;
58   surf_stat_t content = xbt_dict_get(content_dict,path);
59
60   surf_file_t file = xbt_new0(s_surf_file_t,1);
61   file->name = xbt_strdup(path);
62   file->content = content;
63   file->storage = mount;
64
65   surf_action_t action = storage_action_execute(storage,0, OPEN);
66   action->file = (void *)file;
67   return action;
68 }
69
70 static surf_action_t storage_action_close(void *storage, surf_file_t fp)
71 {
72   char *filename = fp->name;
73   free(fp->name);
74   fp->content = NULL;
75   xbt_free(fp);
76   surf_action_t action = storage_action_execute(storage,0, CLOSE);
77   XBT_DEBUG("\tClose file '%s'",filename);
78   return action;
79 }
80
81 static surf_action_t storage_action_read(void *storage, void* ptr, size_t size, size_t nmemb, surf_file_t stream)
82 {
83   char *filename = stream->name;
84   surf_stat_t content = stream->content;
85   XBT_DEBUG("\tRead file '%s' size '%zu/%zu'",filename,size,content->stat.size);
86   if(size > content->stat.size)
87     size = content->stat.size;
88   surf_action_t action = storage_action_execute(storage,size,READ);
89   return action;
90 }
91
92 static surf_action_t storage_action_write(void *storage, const void* ptr, size_t size, size_t nmemb, surf_file_t stream)
93 {
94   char *filename = stream->name;
95   surf_stat_t content = stream->content;
96   XBT_DEBUG("\tWrite file '%s' size '%zu/%zu'",filename,size,content->stat.size);
97
98   // update file size
99   content->stat.size += size;
100   // update disk space
101   ((storage_t)storage)->used_size =+ size;
102
103   surf_action_t action = storage_action_execute(storage,size,WRITE);
104   // if(used_size==size) {
105   //   storage_action_state_set((surf_action_t) action, SURF_ACTION_FAILED);
106   // }
107   return action;
108 }
109
110 static surf_action_t storage_action_execute (void *storage, size_t size, e_surf_action_storage_type_t type)
111 {
112   surf_action_storage_t action = NULL;
113   storage_t STORAGE = storage;
114
115   XBT_IN("(%s,%zu)", surf_resource_name(STORAGE), size);
116   action =
117       surf_action_new(sizeof(s_surf_action_storage_t), size, surf_storage_model,
118           STORAGE->state_current != SURF_RESOURCE_ON);
119
120   GENERIC_LMM_ACTION(action).suspended = 0;     /* Should be useless because of the
121                                                    calloc but it seems to help valgrind... */
122
123   GENERIC_LMM_ACTION(action).variable =
124       lmm_variable_new(storage_maxmin_system, action, 1.0, -1.0 , 3);
125
126   // Must be less than the max bandwidth for all actions
127   lmm_expand(storage_maxmin_system, STORAGE->constraint,
128              GENERIC_LMM_ACTION(action).variable, 1.0);
129
130   switch(type) {
131   case OPEN:
132   case CLOSE:
133   case STAT:
134     break;
135   case READ:
136     lmm_expand(storage_maxmin_system, STORAGE->constraint_read,
137                GENERIC_LMM_ACTION(action).variable, 1.0);
138     break;
139   case WRITE:
140     lmm_expand(storage_maxmin_system, STORAGE->constraint_write,
141                GENERIC_LMM_ACTION(action).variable, 1.0);
142     break;
143   }
144
145   XBT_OUT();
146   return (surf_action_t) action;
147 }
148
149 static void* storage_create_resource(const char* id, const char* model,const char* type_id,const char* content_name)
150 {
151   storage_t storage = NULL;
152
153   xbt_assert(!surf_storage_resource_by_name(id),
154               "Storage '%s' declared several times in the platform file",
155               id);
156   storage = (storage_t) surf_resource_new(sizeof(s_storage_t),
157           surf_storage_model, id,NULL);
158
159   storage->state_current = SURF_RESOURCE_ON;
160   storage->used_size = 0;
161   storage->size = 0;
162
163   storage_type_t storage_type = xbt_lib_get_or_null(storage_type_lib, type_id,ROUTING_STORAGE_TYPE_LEVEL);
164   double Bread  = atof(xbt_dict_get(storage_type->properties,"Bread"));
165   double Bwrite = atof(xbt_dict_get(storage_type->properties,"Bwrite"));
166   double Bconnection   = atof(xbt_dict_get(storage_type->properties,"Bconnection"));
167   XBT_DEBUG("Create resource with Bconnection '%f' Bread '%f' Bwrite '%f' and Size '%ld'",Bconnection,Bread,Bwrite,storage_type->size);
168   storage->constraint       = lmm_constraint_new(storage_maxmin_system, storage, Bconnection);
169   storage->constraint_read  = lmm_constraint_new(storage_maxmin_system, storage, Bread);
170   storage->constraint_write = lmm_constraint_new(storage_maxmin_system, storage, Bwrite);
171   storage->content = parse_storage_content((char*)content_name,&(storage->used_size));
172   storage->size = storage_type->size;
173
174   xbt_lib_set(storage_lib, id, SURF_STORAGE_LEVEL, storage);
175
176   XBT_DEBUG("SURF storage create resource\n\t\tid '%s'\n\t\ttype '%s' \n\t\tmodel '%s' \n\t\tproperties '%p'\n\t\tBread '%f'\n",
177       id,
178       model,
179       type_id,
180       storage_type->properties,
181       Bread);
182
183   return storage;
184 }
185
186 static void storage_finalize(void)
187 {
188   lmm_system_free(storage_maxmin_system);
189   storage_maxmin_system = NULL;
190
191   surf_model_exit(surf_storage_model);
192   surf_storage_model = NULL;
193
194   xbt_swag_free
195       (storage_running_action_set_that_does_not_need_being_checked);
196   storage_running_action_set_that_does_not_need_being_checked = NULL;
197 }
198
199 static void storage_update_actions_state(double now, double delta)
200 {
201   surf_action_storage_t action = NULL;
202   surf_action_storage_t next_action = NULL;
203   xbt_swag_t running_actions = surf_storage_model->states.running_action_set;
204
205   // Update the disk usage
206 //  foreach disk {
207 //    rate = 0
208 //    foreach write_action in disk {
209 //      rate += value(variable(action))
210 //     }
211 //   used_size += delta * rate
212 //  }
213   xbt_swag_foreach_safe(action, next_action, running_actions) {
214     double_update(&(GENERIC_ACTION(action).remains),
215                   lmm_variable_getvalue(GENERIC_LMM_ACTION(action).variable) * delta);
216     if (GENERIC_LMM_ACTION(action).generic_action.max_duration != NO_MAX_DURATION)
217       double_update(&(GENERIC_ACTION(action).max_duration), delta);
218     // if(remains>0 and weight(variable(action))>0 && used_size == size) {
219     // GENERIC_ACTION(action).finish = surf_get_clock();
220     // storage_action_state_set((surf_action_t) action, SURF_ACTION_FAILED);
221     // } else
222     if ((GENERIC_ACTION(action).remains <= 0) &&
223         (lmm_get_variable_weight(GENERIC_LMM_ACTION(action).variable) > 0)) {
224       GENERIC_ACTION(action).finish = surf_get_clock();
225       storage_action_state_set((surf_action_t) action, SURF_ACTION_DONE);
226     } else if ((GENERIC_ACTION(action).max_duration != NO_MAX_DURATION) &&
227                (GENERIC_ACTION(action).max_duration <= 0)) {
228       GENERIC_ACTION(action).finish = surf_get_clock();
229       storage_action_state_set((surf_action_t) action, SURF_ACTION_DONE);
230     }
231   }
232
233   return;
234 }
235
236 static double storage_share_resources(double NOW)
237 {
238   XBT_DEBUG("storage_share_resources %f",NOW);
239   s_surf_action_storage_t action;
240   double min_completion = generic_maxmin_share_resources(surf_storage_model->states.running_action_set,
241       xbt_swag_offset(action, generic_lmm_action.variable),
242       storage_maxmin_system, lmm_solve);
243   /*
244    foreach(disk) {
245       rate = 0
246       foreach write_action in disk {
247           rate += value(variable(action));
248       }
249       if(rate>0) {
250         min_completion = MIN(min_completion, (size-used_size)/rate);
251       }
252   }*/
253
254   return min_completion;
255 }
256
257 static int storage_resource_used(void *resource_id)
258 {
259   THROW_UNIMPLEMENTED;
260   return 0;
261 }
262
263 static void storage_resources_state(void *id, tmgr_trace_event_t event_type,
264                                  double value, double time)
265 {
266   THROW_UNIMPLEMENTED;
267 }
268
269 static int storage_action_unref(surf_action_t action)
270 {
271   action->refcount--;
272   if (!action->refcount) {
273     xbt_swag_remove(action, action->state_set);
274     if (((surf_action_lmm_t) action)->variable)
275       lmm_variable_free(storage_maxmin_system,
276                         ((surf_action_lmm_t) action)->variable);
277 #ifdef HAVE_TRACING
278     xbt_free(action->category);
279 #endif
280     surf_action_free(&action);
281     return 1;
282   }
283   return 0;
284 }
285
286 static void storage_action_cancel(surf_action_t action)
287 {
288   surf_action_state_set(action, SURF_ACTION_FAILED);
289   return;
290 }
291
292 static void storage_action_state_set(surf_action_t action, e_surf_action_state_t state)
293 {
294   surf_action_state_set(action, state);
295   return;
296 }
297
298 static void storage_action_suspend(surf_action_t action)
299 {
300   XBT_IN("(%p)", action);
301   if (((surf_action_lmm_t) action)->suspended != 2) {
302     lmm_update_variable_weight(storage_maxmin_system,
303                                ((surf_action_lmm_t) action)->variable,
304                                0.0);
305     ((surf_action_lmm_t) action)->suspended = 1;
306   }
307   XBT_OUT();
308 }
309
310 static void storage_action_resume(surf_action_t action)
311 {
312   THROW_UNIMPLEMENTED;
313 }
314
315 static int storage_action_is_suspended(surf_action_t action)
316 {
317   return (((surf_action_lmm_t) action)->suspended == 1);
318 }
319
320 static void storage_action_set_max_duration(surf_action_t action, double duration)
321 {
322   THROW_UNIMPLEMENTED;
323 }
324
325 static void storage_action_set_priority(surf_action_t action, double priority)
326 {
327   THROW_UNIMPLEMENTED;
328 }
329
330 static void parse_storage_init(sg_platf_storage_cbarg_t storage)
331 {
332   void* stype = xbt_lib_get_or_null(storage_type_lib,
333                                     storage->type_id,
334                                     ROUTING_STORAGE_TYPE_LEVEL);
335   if(!stype) xbt_die("No storage type '%s'",storage->type_id);
336
337   // if storage content is not specified use the content of storage_type if exist
338   if(!strcmp(storage->content,"") && strcmp(((storage_type_t) stype)->content,"")){
339     storage->content = ((storage_type_t) stype)->content;
340     XBT_DEBUG("For disk '%s' content is empty, use the content of storage type '%s'",storage->id,((storage_type_t) stype)->type_id);
341   }
342
343   XBT_DEBUG("SURF storage create resource\n\t\tid '%s'\n\t\ttype '%s' \n\t\tmodel '%s' \n\t\tcontent '%s'\n\t\tproperties '%p'\n",
344       storage->id,
345       ((storage_type_t) stype)->model,
346       ((storage_type_t) stype)->type_id,
347       storage->content,
348       ((storage_type_t) stype)->properties);
349
350   storage_create_resource(storage->id,
351      ((storage_type_t) stype)->model,
352      ((storage_type_t) stype)->type_id,
353      storage->content);
354 }
355
356 static void parse_mstorage_init(sg_platf_mstorage_cbarg_t mstorage)
357 {
358   XBT_DEBUG("parse_mstorage_init");
359 }
360
361 static void parse_storage_type_init(sg_platf_storage_type_cbarg_t storagetype_)
362 {
363   XBT_DEBUG("parse_storage_type_init");
364 }
365
366 static void parse_mount_init(sg_platf_mount_cbarg_t mount)
367 {
368   XBT_DEBUG("parse_mount_init");
369 }
370
371 static void storage_define_callbacks()
372 {
373   sg_platf_storage_add_cb(parse_storage_init);
374   sg_platf_storage_type_add_cb(parse_storage_type_init);
375   sg_platf_mstorage_add_cb(parse_mstorage_init);
376   sg_platf_mount_add_cb(parse_mount_init);
377 }
378
379 static void surf_storage_model_init_internal(void)
380 {
381   s_surf_action_t action;
382
383   XBT_DEBUG("surf_storage_model_init_internal");
384   surf_storage_model = surf_model_init();
385
386   storage_running_action_set_that_does_not_need_being_checked =
387       xbt_swag_new(xbt_swag_offset(action, state_hookup));
388
389   surf_storage_model->name = "Storage";
390   surf_storage_model->action_unref = storage_action_unref;
391   surf_storage_model->action_cancel = storage_action_cancel;
392   surf_storage_model->action_state_set = storage_action_state_set;
393
394   surf_storage_model->model_private->finalize = storage_finalize;
395   surf_storage_model->model_private->update_actions_state = storage_update_actions_state;
396   surf_storage_model->model_private->share_resources = storage_share_resources;
397   surf_storage_model->model_private->resource_used = storage_resource_used;
398   surf_storage_model->model_private->update_resource_state = storage_resources_state;
399
400   surf_storage_model->suspend = storage_action_suspend;
401   surf_storage_model->resume = storage_action_resume;
402   surf_storage_model->is_suspended = storage_action_is_suspended;
403   surf_storage_model->set_max_duration = storage_action_set_max_duration;
404   surf_storage_model->set_priority = storage_action_set_priority;
405
406   surf_storage_model->extension.storage.open = storage_action_open;
407   surf_storage_model->extension.storage.close = storage_action_close;
408   surf_storage_model->extension.storage.read = storage_action_read;
409   surf_storage_model->extension.storage.write = storage_action_write;
410   surf_storage_model->extension.storage.stat = storage_action_stat;
411   surf_storage_model->extension.storage.create_resource = storage_create_resource;
412
413   if (!storage_maxmin_system) {
414     storage_maxmin_system = lmm_system_new(storage_selective_update);
415   }
416
417 }
418
419 void surf_storage_model_init_default(void)
420 {
421   surf_storage_model_init_internal();
422   storage_define_callbacks();
423
424   xbt_dynar_push(model_list, &surf_storage_model);
425 }
426
427 static void storage_parse_storage(sg_platf_storage_cbarg_t storage)
428 {
429   xbt_assert(!xbt_lib_get_or_null(storage_lib, storage->id,ROUTING_STORAGE_LEVEL),
430                "Reading a storage, processing unit \"%s\" already exists", storage->id);
431
432   // Verification of an existing type_id
433 #ifndef NDEBUG
434   void* storage_type = xbt_lib_get_or_null(storage_type_lib, storage->type_id,ROUTING_STORAGE_TYPE_LEVEL);
435 #endif
436   xbt_assert(storage_type,"Reading a storage, type id \"%s\" does not exists", storage->type_id);
437
438   XBT_DEBUG("ROUTING Create a storage name '%s' with type_id '%s' and content '%s'",
439       storage->id,
440       storage->type_id,
441       storage->content);
442
443   xbt_lib_set(storage_lib,
444       storage->id,
445       ROUTING_STORAGE_LEVEL,
446       (void *) xbt_strdup(storage->type_id));
447 }
448
449 static void free_storage_content(void *p)
450 {
451   surf_stat_t content = p;
452   free(content->stat.date);
453   free(content->stat.group);
454   free(content->stat.time);
455   free(content->stat.user);
456   free(content->stat.user_rights);
457   free(content);
458 }
459
460 static xbt_dict_t parse_storage_content(char *filename, unsigned long *used_size)
461 {
462   *used_size = 0;
463   if ((!filename) || (strcmp(filename, "") == 0))
464     return NULL;
465
466   xbt_dict_t parse_content = xbt_dict_new_homogeneous(free_storage_content);
467   FILE *file = NULL;
468
469   file = surf_fopen(filename, "r");
470   xbt_assert(file != NULL, "Cannot open file '%s' (path=%s)", filename,
471               xbt_str_join(surf_path, ":"));
472
473   char *line = NULL;
474   size_t len = 0;
475   ssize_t read;
476   char user_rights[12];
477   char user[100];
478   char group[100];
479   char date[12];
480   char time[12];
481   char path[1024];
482   int nb;
483   long size;
484
485   surf_stat_t content;
486
487   while ((read = getline(&line, &len, file)) != -1) {
488     content = xbt_new0(s_surf_stat_t,1);
489     if(sscanf(line,"%s %d %s %s %ld %s %s %s",user_rights,&nb,user,group,&size,date,time,path)==8) {
490       content->stat.date = xbt_strdup(date);
491       content->stat.group = xbt_strdup(group);
492       content->stat.size = size;
493       content->stat.time = xbt_strdup(time);
494       content->stat.user = xbt_strdup(user);
495       content->stat.user_rights = xbt_strdup(user_rights);
496       *used_size += content->stat.size;
497       xbt_dict_set(parse_content,path,content,NULL);
498     } else {
499       xbt_die("Be sure of passing a good format for content file.\n");
500       // You can generate this kind of file with command line:
501       // find /path/you/want -type f -exec ls -l {} \; 2>/dev/null > ./content.txt
502     }
503   }
504   if (line)
505       free(line);
506   fclose(file);
507   return parse_content;
508 }
509
510 static void storage_parse_storage_type(sg_platf_storage_type_cbarg_t storage_type)
511 {
512   xbt_assert(!xbt_lib_get_or_null(storage_type_lib, storage_type->id,ROUTING_STORAGE_TYPE_LEVEL),
513                "Reading a storage type, processing unit \"%s\" already exists", storage_type->id);
514
515   storage_type_t stype = xbt_new0(s_storage_type_t, 1);
516   stype->model = xbt_strdup(storage_type->model);
517   stype->properties = storage_type->properties;
518   stype->content = xbt_strdup(storage_type->content);
519   stype->type_id = xbt_strdup(storage_type->id);
520   stype->size = storage_type->size * 1000000000; /* storage_type->size is in Gbytes and stype->sizeis in bytes */
521
522   XBT_DEBUG("ROUTING Create a storage type id '%s' with model '%s' content '%s'",
523       stype->type_id,
524       stype->model,
525       storage_type->content);
526
527   xbt_lib_set(storage_type_lib,
528       stype->type_id,
529       ROUTING_STORAGE_TYPE_LEVEL,
530       (void *) stype);
531 }
532 static void storage_parse_mstorage(sg_platf_mstorage_cbarg_t mstorage)
533 {
534   THROW_UNIMPLEMENTED;
535 //  mount_t mnt = xbt_new0(s_mount_t, 1);
536 //  mnt->id = xbt_strdup(mstorage->type_id);
537 //  mnt->name = xbt_strdup(mstorage->name);
538 //
539 //  if(!mount_list){
540 //    XBT_DEBUG("Creata a Mount list for %s",A_surfxml_host_id);
541 //    mount_list = xbt_dynar_new(sizeof(char *), NULL);
542 //  }
543 //  xbt_dynar_push(mount_list,(void *) mnt);
544 //  free(mnt->id);
545 //  free(mnt->name);
546 //  xbt_free(mnt);
547 //  XBT_DEBUG("ROUTING Mount a storage name '%s' with type_id '%s'",mstorage->name, mstorage->id);
548 }
549
550 static void mount_free(void *p)
551 {
552   mount_t mnt = p;
553   xbt_free(mnt->name);
554 }
555
556 static void storage_parse_mount(sg_platf_mount_cbarg_t mount)
557 {
558   // Verification of an existing storage
559 #ifndef NDEBUG
560   void* storage = xbt_lib_get_or_null(storage_lib, mount->id,ROUTING_STORAGE_LEVEL);
561 #endif
562   xbt_assert(storage,"Disk id \"%s\" does not exists", mount->id);
563
564   XBT_DEBUG("ROUTING Mount '%s' on '%s'",mount->id, mount->name);
565
566   s_mount_t mnt;
567   mnt.id = surf_storage_resource_by_name(mount->id);
568   mnt.name = xbt_strdup(mount->name);
569
570   if(!mount_list){
571     XBT_DEBUG("Create a Mount list for %s",A_surfxml_host_id);
572     mount_list = xbt_dynar_new(sizeof(s_mount_t), mount_free);
573   }
574   xbt_dynar_push(mount_list,&mnt);
575 }
576
577 static XBT_INLINE void routing_storage_type_free(void *r)
578 {
579   storage_type_t stype = r;
580   free(stype->model);
581   free(stype->type_id);
582   free(stype->content);
583   xbt_dict_free(&(stype->properties));
584   free(stype);
585 }
586
587 static XBT_INLINE void surf_storage_resource_free(void *r)
588 {
589   // specific to storage
590   storage_t storage = r;
591   xbt_dict_free(&storage->content);
592   // generic resource
593   surf_resource_free(r);
594 }
595
596 static XBT_INLINE void routing_storage_host_free(void *r)
597 {
598   xbt_dynar_t dyn = r;
599   xbt_dynar_free(&dyn);
600 }
601
602 void storage_register_callbacks() {
603
604   ROUTING_STORAGE_LEVEL = xbt_lib_add_level(storage_lib,xbt_free);
605   ROUTING_STORAGE_HOST_LEVEL = xbt_lib_add_level(storage_lib,routing_storage_host_free);
606   ROUTING_STORAGE_TYPE_LEVEL = xbt_lib_add_level(storage_type_lib,routing_storage_type_free);
607   SURF_STORAGE_LEVEL = xbt_lib_add_level(storage_lib,surf_storage_resource_free);
608
609   sg_platf_storage_add_cb(storage_parse_storage);
610   sg_platf_mstorage_add_cb(storage_parse_mstorage);
611   sg_platf_storage_type_add_cb(storage_parse_storage_type);
612   sg_platf_mount_add_cb(storage_parse_mount);
613 }