Logo AND Algorithmique Numérique Distribuée

Public GIT Repository
Reorganize storage structures (msg, smx and surf) and separate storage parsing from...
[simgrid.git] / src / surf / storage.c
1 /* Copyright (c) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012. The SimGrid Team.
2  * All rights reserved.                                                                 */
3
4 /* This program is free software; you can redistribute it and/or modify it
5  * under the terms of the license (GNU LGPL) which comes with this package.             */
6
7 #include "xbt/ex.h"
8 #include "xbt/dict.h"
9 #include "portable.h"
10 #include "surf_private.h"
11 #include "storage_private.h"
12 #include "surf/surf_resource.h"
13
14 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(surf_storage, surf,
15                                 "Logging specific to the SURF storage module");
16
17 xbt_lib_t storage_lib;
18 int ROUTING_STORAGE_LEVEL;      //Routing for storagelevel
19 int ROUTING_STORAGE_HOST_LEVEL;
20 int SURF_STORAGE_LEVEL;
21 xbt_lib_t storage_type_lib;
22 int ROUTING_STORAGE_TYPE_LEVEL; //Routing for storage_type level
23
24 xbt_dynar_t mount_list = NULL;  /* temporary store of current mount storage */
25
26 surf_model_t surf_storage_model = NULL;
27 lmm_system_t storage_maxmin_system = NULL;
28 static int storage_selective_update = 0;
29 static xbt_swag_t
30     storage_running_action_set_that_does_not_need_being_checked = NULL;
31
32 #define GENERIC_LMM_ACTION(action) action->generic_lmm_action
33 #define GENERIC_ACTION(action) GENERIC_LMM_ACTION(action).generic_action
34
35 typedef enum {
36   READ = 0,
37   WRITE = 1,
38   DEFAULT = 2
39 } STORAGE_execute_type_t;
40
41 typedef struct surf_storage {
42   s_surf_resource_t generic_resource;
43   const char* type;
44   const char* content; /*should be a dict */
45 } s_surf_storage_t, *surf_storage_t;
46
47 static void storage_action_state_set(surf_action_t action, e_surf_action_state_t state);
48 static surf_action_t storage_action_sleep (void *storage, double duration);
49 static surf_action_t storage_action_execute (void *storage, size_t size, STORAGE_execute_type_t type);
50
51 static surf_action_t storage_action_open(void *storage, const char* path, const char* mode)
52 {
53   XBT_DEBUG("\tOpen file '%s'",path);
54   char *storage_type_id = xbt_lib_get_or_null(
55       storage_lib,
56       ((storage_t)storage)->generic_resource.name,
57       ROUTING_STORAGE_LEVEL);
58   storage_type_t storage_type = xbt_lib_get_or_null(storage_type_lib, storage_type_id,ROUTING_STORAGE_TYPE_LEVEL);
59   xbt_dict_t content_dict = storage_type->content;
60   content_t content = xbt_dict_get(content_dict,path);
61
62   surf_file_t file = xbt_new0(s_surf_file_t,1);
63   file->name = xbt_strdup(path);
64   file->content = content;
65
66   surf_action_t action = storage_action_execute(storage,0, DEFAULT);
67   action->file = (void *)file;
68   return action;
69 }
70
71 static surf_action_t storage_action_close(void *storage, surf_file_t fp)
72 {
73   char *filename = fp->name;
74   free(fp->name);
75   fp->content = NULL;
76   xbt_free(fp);
77   surf_action_t action = storage_action_execute(storage,0, DEFAULT);
78   XBT_DEBUG("\tClose file '%s'",filename);
79   return action;
80 }
81
82 static surf_action_t storage_action_read(void *storage, void* ptr, size_t size, size_t nmemb, surf_file_t stream)
83 {
84   char *filename = stream->name;
85   content_t content = stream->content;
86   XBT_DEBUG("\tRead file '%s' size '%Zu/%Zu'",filename,size,content->size);
87   if(size > content->size)
88     size = content->size;
89   surf_action_t action = storage_action_execute(storage,size,READ);
90   action->read_write = size;
91   return action;
92 }
93
94 static surf_action_t storage_action_write(void *storage, const void* ptr, size_t size, size_t nmemb, surf_file_t stream)
95 {
96   char *filename = stream->name;
97   content_t content = stream->content;
98   XBT_DEBUG("\tWrite file '%s' size '%Zu/%Zu'",filename,size,content->size);
99
100   surf_action_t action = storage_action_execute(storage,size,WRITE);
101   action->read_write = size;
102   content->size += size;
103   return action;
104 }
105
106 static surf_action_t storage_action_stat(void *storage, int fd, void* buf)
107 {
108   THROW_UNIMPLEMENTED;
109   return NULL;
110 }
111
112 static surf_action_t storage_action_execute (void *storage, size_t size, STORAGE_execute_type_t type)
113 {
114   surf_action_storage_t action = NULL;
115   storage_t STORAGE = storage;
116
117   XBT_IN("(%s,%Zu)", surf_resource_name(STORAGE), size);
118   action =
119       surf_action_new(sizeof(s_surf_action_storage_t), size, surf_storage_model,
120           STORAGE->state_current != SURF_RESOURCE_ON);
121
122   GENERIC_LMM_ACTION(action).suspended = 0;     /* Should be useless because of the
123                                                    calloc but it seems to help valgrind... */
124
125   GENERIC_LMM_ACTION(action).variable =
126       lmm_variable_new(storage_maxmin_system, action, 1.0, -1.0 , 3);
127
128   if(type == DEFAULT)
129   lmm_expand(storage_maxmin_system, STORAGE->constraint,
130              GENERIC_LMM_ACTION(action).variable, 1.0);
131   else if(type == READ)
132     lmm_expand(storage_maxmin_system, STORAGE->constraint_read,
133                GENERIC_LMM_ACTION(action).variable, 1.0);
134   else if(type == WRITE)
135     lmm_expand(storage_maxmin_system, STORAGE->constraint_write,
136                GENERIC_LMM_ACTION(action).variable, 1.0);
137   else xbt_die("storage_action_execute with type '%d'",(int) type);
138
139   XBT_OUT();
140   return (surf_action_t) action;
141 }
142
143 static surf_action_t storage_action_sleep (void *storage, double duration)
144 {
145   surf_action_storage_t action = NULL;
146
147   if (duration > 0)
148     duration = MAX(duration, MAXMIN_PRECISION);
149
150   XBT_IN("(%s,%g)", surf_resource_name(storage), duration);
151   action = (surf_action_storage_t) storage_action_execute(storage, 1.0, DEFAULT);
152   GENERIC_ACTION(action).max_duration = duration;
153   GENERIC_LMM_ACTION(action).suspended = 2;
154   if (duration == NO_MAX_DURATION) {
155     /* Move to the *end* of the corresponding action set. This convention
156        is used to speed up update_resource_state  */
157     xbt_swag_remove(action, ((surf_action_t) action)->state_set);
158     ((surf_action_t) action)->state_set =
159         storage_running_action_set_that_does_not_need_being_checked;
160     xbt_swag_insert(action, ((surf_action_t) action)->state_set);
161   }
162
163   lmm_update_variable_weight(storage_maxmin_system,
164                              GENERIC_LMM_ACTION(action).variable, 0.0);
165
166   XBT_OUT();
167   return (surf_action_t) action;
168 }
169
170 static void* storage_create_resource(const char* id, const char* model,const char* type_id)
171 {
172   storage_t storage = NULL;
173
174   xbt_assert(!surf_storage_resource_by_name(id),
175               "Storage '%s' declared several times in the platform file",
176               id);
177   storage = (storage_t) surf_resource_new(sizeof(s_storage_t),
178           surf_storage_model, id,NULL);
179
180   storage->state_current = SURF_RESOURCE_ON;
181
182   storage_type_t storage_type = xbt_lib_get_or_null(storage_type_lib, type_id,ROUTING_STORAGE_TYPE_LEVEL);
183   size_t Bread  = atof(xbt_dict_get(storage_type->properties,"Bread"));
184   size_t Bwrite = atof(xbt_dict_get(storage_type->properties,"Bwrite"));
185   size_t Bconnexion   = atof(xbt_dict_get(storage_type->properties,"Bconnexion"));
186   XBT_INFO("Create resource with Bconnexion '%Zu' Bread '%Zu' Bwrite '%Zu'",Bconnexion,Bread,Bwrite);
187   storage->constraint       = lmm_constraint_new(storage_maxmin_system, storage, Bconnexion);
188   storage->constraint_read  = lmm_constraint_new(storage_maxmin_system, storage, Bread);
189   storage->constraint_write = lmm_constraint_new(storage_maxmin_system, storage, Bwrite);
190
191   xbt_lib_set(storage_lib, id, SURF_STORAGE_LEVEL, storage);
192
193   XBT_DEBUG("SURF storage create resource\n\t\tid '%s'\n\t\ttype '%s' \n\t\tmodel '%s' \n\t\tproperties '%p'\n\t\tBread '%Zu'\n",
194       id,
195       model,
196       type_id,
197       storage_type->properties,
198       Bread);
199
200   return storage;
201 }
202
203 static void storage_finalize(void)
204 {
205   lmm_system_free(storage_maxmin_system);
206   storage_maxmin_system = NULL;
207
208   surf_model_exit(surf_storage_model);
209   surf_storage_model = NULL;
210
211   xbt_swag_free
212       (storage_running_action_set_that_does_not_need_being_checked);
213   storage_running_action_set_that_does_not_need_being_checked = NULL;
214 }
215
216 static void storage_update_actions_state(double now, double delta)
217 {
218   surf_action_storage_t action = NULL;
219   surf_action_storage_t next_action = NULL;
220   xbt_swag_t running_actions = surf_storage_model->states.running_action_set;
221   xbt_swag_foreach_safe(action, next_action, running_actions) {
222
223     double_update(&(GENERIC_ACTION(action).remains),
224                   lmm_variable_getvalue(GENERIC_LMM_ACTION(action).variable) * delta);
225     if (GENERIC_LMM_ACTION(action).generic_action.max_duration != NO_MAX_DURATION)
226       double_update(&(GENERIC_ACTION(action).max_duration), delta);
227     if ((GENERIC_ACTION(action).remains <= 0) &&
228         (lmm_get_variable_weight(GENERIC_LMM_ACTION(action).variable) > 0)) {
229       GENERIC_ACTION(action).finish = surf_get_clock();
230       storage_action_state_set((surf_action_t) action, SURF_ACTION_DONE);
231     } else if ((GENERIC_ACTION(action).max_duration != NO_MAX_DURATION) &&
232                (GENERIC_ACTION(action).max_duration <= 0)) {
233       GENERIC_ACTION(action).finish = surf_get_clock();
234       storage_action_state_set((surf_action_t) action, SURF_ACTION_DONE);
235     }
236   }
237
238   return;
239 }
240
241 static double storage_share_resources(double NOW)
242 {
243   XBT_DEBUG("storage_share_resources %f",NOW);
244   s_surf_action_storage_t action;
245   return generic_maxmin_share_resources(surf_storage_model->states.running_action_set,
246       xbt_swag_offset(action, generic_lmm_action.variable),
247       storage_maxmin_system, lmm_solve);
248 }
249
250 static int storage_resource_used(void *resource_id)
251 {
252   THROW_UNIMPLEMENTED;
253   return 0;
254 }
255
256 static void storage_resources_state(void *id, tmgr_trace_event_t event_type,
257                                  double value, double time)
258 {
259   THROW_UNIMPLEMENTED;
260 }
261
262 static int storage_action_unref(surf_action_t action)
263 {
264   action->refcount--;
265   if (!action->refcount) {
266     xbt_swag_remove(action, action->state_set);
267     if (((surf_action_lmm_t) action)->variable)
268       lmm_variable_free(storage_maxmin_system,
269                         ((surf_action_lmm_t) action)->variable);
270 #ifdef HAVE_TRACING
271     xbt_free(action->category);
272 #endif
273     surf_action_free(&action);
274     return 1;
275   }
276   return 0;
277 }
278
279 static void storage_action_cancel(surf_action_t action)
280 {
281   surf_action_state_set(action, SURF_ACTION_FAILED);
282   return;
283 }
284
285 static void storage_action_state_set(surf_action_t action, e_surf_action_state_t state)
286 {
287   surf_action_state_set(action, state);
288   return;
289 }
290
291 static void storage_action_suspend(surf_action_t action)
292 {
293   XBT_IN("(%p)", action);
294   if (((surf_action_lmm_t) action)->suspended != 2) {
295     lmm_update_variable_weight(storage_maxmin_system,
296                                ((surf_action_lmm_t) action)->variable,
297                                0.0);
298     ((surf_action_lmm_t) action)->suspended = 1;
299   }
300   XBT_OUT();
301 }
302
303 static void storage_action_resume(surf_action_t action)
304 {
305   THROW_UNIMPLEMENTED;
306 }
307
308 static int storage_action_is_suspended(surf_action_t action)
309 {
310   return (((surf_action_lmm_t) action)->suspended == 1);
311 }
312
313 static void storage_action_set_max_duration(surf_action_t action, double duration)
314 {
315   THROW_UNIMPLEMENTED;
316 }
317
318 static void storage_action_set_priority(surf_action_t action, double priority)
319 {
320   THROW_UNIMPLEMENTED;
321 }
322
323 static void parse_storage_init(sg_platf_storage_cbarg_t storage)
324 {
325   void* stype = xbt_lib_get_or_null(storage_type_lib,
326                                     storage->type_id,
327                                     ROUTING_STORAGE_TYPE_LEVEL);
328   if(!stype) xbt_die("No storage type '%s'",storage->type_id);
329 //  XBT_INFO("SURF storage create resource\n\t\tid '%s'\n\t\ttype '%s' \n\t\tmodel '%s' \n\t\tcontent '%s'\n\t\tproperties '%p'\n",
330 //      storage->id,
331 //      ((storage_type_t) stype)->model,
332 //      ((storage_type_t) stype)->type_id,
333 //      ((storage_type_t) stype)->content,
334 //      ((storage_type_t) stype)->properties);
335
336  storage_create_resource(storage->id,
337      ((storage_type_t) stype)->model,
338      ((storage_type_t) stype)->type_id);
339 }
340
341 static void parse_mstorage_init(sg_platf_mstorage_cbarg_t mstorage)
342 {
343   XBT_DEBUG("parse_mstorage_init");
344 }
345
346 static void parse_storage_type_init(sg_platf_storage_type_cbarg_t storagetype_)
347 {
348   XBT_DEBUG("parse_storage_type_init");
349 }
350
351 static void parse_mount_init(sg_platf_mount_cbarg_t mount)
352 {
353   XBT_DEBUG("parse_mount_init");
354 }
355
356 static void storage_define_callbacks()
357 {
358   sg_platf_storage_add_cb(parse_storage_init);
359   sg_platf_storage_type_add_cb(parse_storage_type_init);
360   sg_platf_mstorage_add_cb(parse_mstorage_init);
361   sg_platf_mount_add_cb(parse_mount_init);
362 }
363
364 static void surf_storage_model_init_internal(void)
365 {
366   s_surf_action_t action;
367
368   XBT_DEBUG("surf_storage_model_init_internal");
369   surf_storage_model = surf_model_init();
370
371   storage_running_action_set_that_does_not_need_being_checked =
372       xbt_swag_new(xbt_swag_offset(action, state_hookup));
373
374   surf_storage_model->name = "Storage";
375   surf_storage_model->action_unref = storage_action_unref;
376   surf_storage_model->action_cancel = storage_action_cancel;
377   surf_storage_model->action_state_set = storage_action_state_set;
378
379   surf_storage_model->model_private->finalize = storage_finalize;
380   surf_storage_model->model_private->update_actions_state = storage_update_actions_state;
381   surf_storage_model->model_private->share_resources = storage_share_resources;
382   surf_storage_model->model_private->resource_used = storage_resource_used;
383   surf_storage_model->model_private->update_resource_state = storage_resources_state;
384
385   surf_storage_model->suspend = storage_action_suspend;
386   surf_storage_model->resume = storage_action_resume;
387   surf_storage_model->is_suspended = storage_action_is_suspended;
388   surf_storage_model->set_max_duration = storage_action_set_max_duration;
389   surf_storage_model->set_priority = storage_action_set_priority;
390
391   surf_storage_model->extension.storage.open = storage_action_open;
392   surf_storage_model->extension.storage.close = storage_action_close;
393   surf_storage_model->extension.storage.read = storage_action_read;
394   surf_storage_model->extension.storage.write = storage_action_write;
395   surf_storage_model->extension.storage.stat = storage_action_stat;
396   surf_storage_model->extension.storage.create_resource = storage_create_resource;
397
398   if (!storage_maxmin_system) {
399     storage_maxmin_system = lmm_system_new(storage_selective_update);
400   }
401
402 }
403
404 void surf_storage_model_init_default(void)
405 {
406   surf_storage_model_init_internal();
407   storage_define_callbacks();
408
409   xbt_dynar_push(model_list, &surf_storage_model);
410 }
411
412 static void storage_parse_storage(sg_platf_storage_cbarg_t storage)
413 {
414   xbt_assert(!xbt_lib_get_or_null(storage_lib, storage->id,ROUTING_STORAGE_LEVEL),
415                "Reading a storage, processing unit \"%s\" already exists", storage->id);
416
417   // Verification of an existing type_id
418 #ifndef NDEBUG
419   void* storage_type = xbt_lib_get_or_null(storage_type_lib, storage->type_id,ROUTING_STORAGE_TYPE_LEVEL);
420 #endif
421   xbt_assert(storage_type,"Reading a storage, type id \"%s\" does not exists", storage->type_id);
422
423   XBT_DEBUG("ROUTING Create a storage name '%s' with type_id '%s'",
424       storage->id,
425       storage->type_id);
426
427   xbt_lib_set(storage_lib,
428       storage->id,
429       ROUTING_STORAGE_LEVEL,
430       (void *) xbt_strdup(storage->type_id));
431 }
432
433 static void free_storage_content(void *p)
434 {
435   content_t content = p;
436   free(content->date);
437   free(content->group);
438   free(content->time);
439   free(content->user);
440   free(content->user_rights);
441   free(content);
442 }
443
444 static xbt_dict_t parse_storage_content(const char *filename)
445 {
446   if ((!filename) || (strcmp(filename, "") == 0))
447     return NULL;
448
449   xbt_dict_t parse_content = xbt_dict_new_homogeneous(free_storage_content);
450   FILE *file = NULL;
451
452   file = surf_fopen(filename, "r");
453   xbt_assert(file != NULL, "Cannot open file '%s' (path=%s)", filename,
454               xbt_str_join(surf_path, ":"));
455
456   char *line = NULL;
457   size_t len = 0;
458   ssize_t read;
459   char user_rights[12];
460   char user[100];
461   char group[100];
462   char date[12];
463   char time[12];
464   char path[1024];
465   int nb, size;
466
467   content_t content;
468
469   while ((read = getline(&line, &len, file)) != -1) {
470     content = xbt_new0(s_content_t,1);
471     if(sscanf(line,"%s %d %s %s %d %s %s %s",user_rights,&nb,user,group,&size,date,time,path)==8) {
472       content->date = xbt_strdup(date);
473       content->group = xbt_strdup(group);
474       content->size = size;
475       content->time = xbt_strdup(time);
476       content->user = xbt_strdup(user);
477       content->user_rights = xbt_strdup(user_rights);
478       xbt_dict_set(parse_content,path,content,NULL);
479     } else {
480       xbt_die("Be sure of passing a good format for content file.\n");
481       // You can generate this kind of file with command line:
482       // find /path/you/want -type f -exec ls -l {} \; 2>/dev/null > ./content.txt
483     }
484   }
485   if (line)
486       free(line);
487
488   fclose(file);
489   return parse_content;
490 }
491
492 static void storage_parse_storage_type(sg_platf_storage_type_cbarg_t storage_type)
493 {
494   xbt_assert(!xbt_lib_get_or_null(storage_type_lib, storage_type->id,ROUTING_STORAGE_TYPE_LEVEL),
495                "Reading a storage type, processing unit \"%s\" already exists", storage_type->id);
496
497   storage_type_t stype = xbt_new0(s_storage_type_t, 1);
498   stype->model = xbt_strdup(storage_type->model);
499   stype->properties = storage_type->properties;
500   stype->content = parse_storage_content(storage_type->content);
501   stype->type_id = xbt_strdup(storage_type->id);
502
503   XBT_DEBUG("ROUTING Create a storage type id '%s' with model '%s' content '%s' and properties '%p'",
504       stype->type_id,
505       stype->model,
506       storage_type->content,
507       stype->properties);
508
509   xbt_lib_set(storage_type_lib,
510       stype->type_id,
511       ROUTING_STORAGE_TYPE_LEVEL,
512       (void *) stype);
513 }
514 static void storage_parse_mstorage(sg_platf_mstorage_cbarg_t mstorage)
515 {
516   THROW_UNIMPLEMENTED;
517 //  mount_t mnt = xbt_new0(s_mount_t, 1);
518 //  mnt->id = xbt_strdup(mstorage->type_id);
519 //  mnt->name = xbt_strdup(mstorage->name);
520 //
521 //  if(!mount_list){
522 //    XBT_DEBUG("Creata a Mount list for %s",A_surfxml_host_id);
523 //    mount_list = xbt_dynar_new(sizeof(char *), NULL);
524 //  }
525 //  xbt_dynar_push(mount_list,(void *) mnt);
526 //  free(mnt->id);
527 //  free(mnt->name);
528 //  xbt_free(mnt);
529 //  XBT_DEBUG("ROUTING Mount a storage name '%s' with type_id '%s'",mstorage->name, mstorage->id);
530 }
531
532 static void mount_free(void *p)
533 {
534   mount_t mnt = p;
535   xbt_free(mnt->name);
536 }
537
538 static void storage_parse_mount(sg_platf_mount_cbarg_t mount)
539 {
540   // Verification of an existing storage
541 #ifndef NDEBUG
542   void* storage = xbt_lib_get_or_null(storage_lib, mount->id,ROUTING_STORAGE_LEVEL);
543 #endif
544   xbt_assert(storage,"Disk id \"%s\" does not exists", mount->id);
545
546   XBT_DEBUG("ROUTING Mount '%s' on '%s'",mount->id, mount->name);
547
548   s_mount_t mnt;
549   mnt.id = surf_storage_resource_by_name(mount->id);
550   mnt.name = xbt_strdup(mount->name);
551
552   if(!mount_list){
553     XBT_DEBUG("Create a Mount list for %s",A_surfxml_host_id);
554     mount_list = xbt_dynar_new(sizeof(s_mount_t), mount_free);
555   }
556   xbt_dynar_push(mount_list,&mnt);
557 }
558
559 static XBT_INLINE void routing_storage_type_free(void *r)
560 {
561   storage_type_t stype = r;
562   free(stype->model);
563   free(stype->type_id);
564   xbt_dict_free(&(stype->properties));
565   xbt_dict_free(&(stype->content));
566   free(stype);
567 }
568
569 static XBT_INLINE void routing_storage_host_free(void *r)
570 {
571   xbt_dynar_t dyn = r;
572   xbt_dynar_free(&dyn);
573 }
574
575 void storage_register_callbacks() {
576
577   ROUTING_STORAGE_LEVEL = xbt_lib_add_level(storage_lib,xbt_free);
578   ROUTING_STORAGE_HOST_LEVEL = xbt_lib_add_level(storage_lib,routing_storage_host_free);
579   ROUTING_STORAGE_TYPE_LEVEL = xbt_lib_add_level(storage_type_lib,routing_storage_type_free);
580   SURF_STORAGE_LEVEL = xbt_lib_add_level(storage_lib,surf_resource_free);
581
582   sg_platf_storage_add_cb(storage_parse_storage);
583   sg_platf_mstorage_add_cb(storage_parse_mstorage);
584   sg_platf_storage_type_add_cb(storage_parse_storage_type);
585   sg_platf_mount_add_cb(storage_parse_mount);
586 }