X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/375f3e43fa33308a125965ea42e32dfc95339a91..82ef813a5d1cf608e1ae2a80c306a1cee9fceab1:/examples/gras/pmm/pmm.c

diff --git a/examples/gras/pmm/pmm.c b/examples/gras/pmm/pmm.c
index 4e398b8530..45f427e39b 100755
--- a/examples/gras/pmm/pmm.c
+++ b/examples/gras/pmm/pmm.c
@@ -1,58 +1,60 @@
 /* $Id$ */
-/* pmm - parallel matrix multiplication "double diffusion"                       */
+/* pmm - parallel matrix multiplication "double diffusion"                  */
 
-/* Copyright (c) 2006- Ahmed Harbaoui. All rights reserved.                  */
+/* Copyright (c) 2006 Ahmed Harbaoui.                                       */
+/* Copyright (c) 2006 Martin Quinson.                                       */
+/* All rights reserved.                                                     */
 
 /* This program is free software; you can redistribute it and/or modify it
  * under the terms of the license (GNU LGPL) which comes with this package. */
 
 #include "gras.h"
-#define PROC_MATRIX_SIZE 3
+#include "xbt/matrix.h"
+#define PROC_MATRIX_SIZE 2
 #define SLAVE_COUNT (PROC_MATRIX_SIZE*PROC_MATRIX_SIZE)
 
-#define DATA_MATRIX_SIZE 3
+#define DATA_MATRIX_SIZE 8
+const int submatrix_size = DATA_MATRIX_SIZE/PROC_MATRIX_SIZE;
 
 XBT_LOG_NEW_DEFAULT_CATEGORY(pmm,"Parallel Matrix Multiplication");
 
-GRAS_DEFINE_TYPE(s_matrix,struct s_matrix {
-  int rows;
-  int cols;
-  double *data GRAS_ANNOTE(size, rows*cols);
-};)
-typedef struct s_matrix matrix_t;
-
 /* struct for recovering results */
 GRAS_DEFINE_TYPE(s_result,struct s_result {
-  int i;
-  int j;
-  double value;
+  int linepos;
+  int rowpos;
+  xbt_matrix_t C GRAS_ANNOTE(subtype,double);
 });
 typedef struct s_result result_t;
 
 /* struct to send initial data to slave */
-GRAS_DEFINE_TYPE(s_init_data,struct s_init_data {
-  int myrow;
-  int mycol;
-  double a;
-  double b;
+GRAS_DEFINE_TYPE(s_assignment,struct s_assignment {
+  int linepos;
+  int rowpos;
+  xbt_host_t line[PROC_MATRIX_SIZE];
+  xbt_host_t row[PROC_MATRIX_SIZE];
+  xbt_matrix_t A GRAS_ANNOTE(subtype,double);
+  xbt_matrix_t B GRAS_ANNOTE(subtype,double);
 });
-typedef struct s_init_data init_data_t;
+typedef struct s_assignment s_assignment_t;
 
 /* register messages which may be sent (common to client and server) */
 static void register_messages(void) {
   gras_datadesc_type_t result_type;
-  gras_datadesc_type_t init_data_type;
+  gras_datadesc_type_t assignment_type;
+
+  gras_datadesc_set_const("PROC_MATRIX_SIZE",PROC_MATRIX_SIZE);
   result_type=gras_datadesc_by_symbol(s_result);
-  init_data_type=gras_datadesc_by_symbol(s_init_data);
+  assignment_type=gras_datadesc_by_symbol(s_assignment);
 	
-  gras_msgtype_declare("result", result_type);  // receive a final result from slave
-  gras_msgtype_declare("init_data", init_data_type);  // send from master to slave to initialize data bA,bB
-
-  gras_msgtype_declare("ask_result", gras_datadesc_by_name("int")); // send from master to slave to ask a final result	
-  gras_msgtype_declare("step", gras_datadesc_by_name("int"));// send from master to slave to indicate the begining of step 
-  gras_msgtype_declare("step_ack", gras_datadesc_by_name("int"));//send from slave to master to indicate the end of the current step
-  gras_msgtype_declare("dataA", gras_datadesc_by_name("double"));// send data between slave
-  gras_msgtype_declare("dataB", gras_datadesc_by_name("double"));// send data between slave
+  /* receive a final result from slave */
+  gras_msgtype_declare("result", result_type);
+
+  /* send from master to slave to assign a position and some data */
+  gras_msgtype_declare("assignment", assignment_type);
+
+  /* send data between slaves */
+  gras_msgtype_declare("dataA", gras_datadesc_matrix(gras_datadesc_by_name("double"),NULL));
+  gras_msgtype_declare("dataB", gras_datadesc_matrix(gras_datadesc_by_name("double"),NULL));
 }
 
 /* Function prototypes */
@@ -66,21 +68,12 @@ int master (int argc,char *argv[]);
 
 /* Global private data */
 typedef struct {
-  int nbr_col,nbr_row;
+  int nbr_row,nbr_line;
   int remaining_step;
   int remaining_ack;
 } master_data_t;
 
 
-/***  Function initilaze matrixs ***/
-
-static void initmatrix(matrix_t *X){
-  int i;
-
-  for(i=0 ; i<(X->rows)*(X->cols); i++)
-    X->data[i]=1.0;//*rand()/(RAND_MAX+1.0);
-} /* end_of_initmatrixs */
-
 /***  Function Scatter Sequentiel ***/
 
 static void scatter(){
@@ -105,62 +98,27 @@ static void gather(){
 
 }/* end_of_gather */
 
-/***  Function: Display Matrix ***/
-
-static void display(matrix_t X){
-	
-  int i,j,t=0;
-
-  printf("      ");
-  for(j=0;j<X.cols;j++)
-    printf("%.3d ",j);
-  printf("\n");
-  printf("    __");
-  for(j=0;j<X.cols;j++)
-    printf("____");
-  printf("_\n");
-
-  for(i=0;i<X.rows;i++){
-    printf("%.3d | ",i);
-    for(j=0;j<X.cols;j++)
-      printf("%.3g ",X.data[t++]);
-    printf("|\n");
-  }
-  printf("    --");
-  for(j=0;j<X.cols;j++)
-    printf("----");
-  printf("-\n");
-
-}/* end_of_display */
-
 int master (int argc,char *argv[]) {
 
-  xbt_ex_t e;
+  int i,port;
 
-  int i,port,ask_result,step;
-
-  matrix_t A,B,C;
+  xbt_matrix_t A,B,C;
   result_t result;
 
   gras_socket_t from;
 
-  /*  Init the GRAS's infrastructure */
-  gras_init(&argc, argv);
 
+  xbt_host_t grid[SLAVE_COUNT]; /* The slaves */
   gras_socket_t socket[SLAVE_COUNT]; /* sockets for brodcast to slaves */
 
-  /*  Initialize Matrixs */
-
-  A.rows=A.cols=DATA_MATRIX_SIZE;
-  B.rows=B.cols=DATA_MATRIX_SIZE;
-  C.rows=C.cols=DATA_MATRIX_SIZE;
-	
-  A.data=xbt_malloc0(sizeof(double)*DATA_MATRIX_SIZE*DATA_MATRIX_SIZE);
-  B.data=xbt_malloc0(sizeof(double)*DATA_MATRIX_SIZE*DATA_MATRIX_SIZE);
-  C.data=xbt_malloc0(sizeof(double)*DATA_MATRIX_SIZE*DATA_MATRIX_SIZE);
-	
-  initmatrix(&A);
-  initmatrix(&B);
+  /*  Init the GRAS's infrastructure */
+  gras_init(&argc, argv);
+  register_messages();
+      
+  /*  Initialize Matrices */
+  A = xbt_matrix_double_new_id(DATA_MATRIX_SIZE,DATA_MATRIX_SIZE);
+  B = xbt_matrix_double_new_seq(DATA_MATRIX_SIZE,DATA_MATRIX_SIZE);
+  C = xbt_matrix_double_new_zeros(DATA_MATRIX_SIZE,DATA_MATRIX_SIZE);
 	
   /*  Get arguments and create sockets */
   port=atoi(argv[1]);
@@ -168,84 +126,82 @@ int master (int argc,char *argv[]) {
   //scatter_parl();
   //multiplication();
   //gather();
-  //display(A);
   /************************* Init Data Send *********************************/
-  int step_ack,j=0;
-  init_data_t mydata;
-  gras_os_sleep(60);      // MODIFIER LES TEMPS D'ATTENTE 60 c trop normalement
-
-  int row=1, col=1;
-  for( i=2;i< argc;i++){
-    TRY {
-      socket[j]=gras_socket_client(argv[i],port);
-    } CATCH(e) {
-      RETHROW0("Unable to connect to the server: %s");
+  gras_os_sleep(2);
+
+  for( i=1;i<argc && i<=SLAVE_COUNT;i++){
+    grid[i-1]=xbt_host_from_string(argv[i]);
+    socket[i-1]=gras_socket_client(grid[i-1]->name,grid[i-1]->port);
+      
+    INFO2("Connected to %s:%d.",grid[i-1]->name,grid[i-1]->port);
+  }
+  xbt_assert2(i-1==SLAVE_COUNT,
+	      "Not enough slaves for this setting (got %d of %d). Change the deployment file",
+	      i-1,SLAVE_COUNT);
+  /* FIXME: let the surnumerous slave die properly */
+ 
+  int row=0, line=0;
+  INFO0("XXXXXXXXXXXXXXXXXXXXXX begin Multiplication");
+  for(i=0 ; i<SLAVE_COUNT; i++){
+    s_assignment_t assignment;
+    int j;
+
+    assignment.linepos=line; // assigned line
+    assignment.rowpos=row;   // assigned row
+
+    /* Neiborhood */
+    for (j=0; j<PROC_MATRIX_SIZE; j++) {
+      assignment.row[j] = grid[ j*PROC_MATRIX_SIZE+(row) ] ;
+      assignment.line[j] =  grid[ (line)*PROC_MATRIX_SIZE+j ] ;
     }
-    INFO2("Connected to %s:%d.",argv[i],port);
-		
-    mydata.myrow=row;  // My row
-    mydata.mycol=col;  // My column
+
+    assignment.A=xbt_matrix_new_sub(A,
+				    submatrix_size,submatrix_size,
+				    submatrix_size*line,submatrix_size*row,
+				    NULL);
+    assignment.B=xbt_matrix_new_sub(B,
+				    submatrix_size,submatrix_size,
+				    submatrix_size*line,submatrix_size*row,
+				    NULL);
     row++;
-    if (row > PROC_MATRIX_SIZE) {
-      row=1;
-      col++;
+    if (row >= PROC_MATRIX_SIZE) {
+      row=0;
+      line++;
     }
 		
-    mydata.a=A.data[(mydata.myrow-1)*PROC_MATRIX_SIZE+(mydata.mycol-1)];
-    mydata.b=B.data[(mydata.myrow-1)*PROC_MATRIX_SIZE+(mydata.mycol-1)];;
-		
-    gras_msg_send(socket[j],gras_msgtype_by_name("init_data"),&mydata);
-    INFO3("Send Init Data to %s : data A= %.3g & data B= %.3g",
-	  gras_socket_peer_name(socket[j]),mydata.a,mydata.b);
-    j++;
+    gras_msg_send(socket[i],gras_msgtype_by_name("assignment"),&assignment);
+    xbt_matrix_free(assignment.A);
+    xbt_matrix_free(assignment.B);
   }
-    // end init Data Send
+  // end assignment
 
   /******************************* multiplication ********************************/
-  INFO0("XXXXXXXXXXXXXXXXXXXXXX begin Multiplication");
-	
-  for (step=1; step <= PROC_MATRIX_SIZE; step++){
-    //    gras_os_sleep(50);
-    for (i=0; i< SLAVE_COUNT; i++){
-      TRY {
-	gras_msg_send(socket[i], gras_msgtype_by_name("step"), &step);
-      } CATCH(e) {
-	gras_socket_close(socket[i]);
-	RETHROW0("Unable to send the msg : %s");
-      }
-    }
-    INFO1("send to slave to begin a %d th step",step);
-    /* wait for computing and slave messages exchange */
-    i=0;
-	
-    while  ( i< SLAVE_COUNT){
-      TRY {
-	gras_msg_wait(1300,gras_msgtype_by_name("step_ack"),&from,&step_ack);
-      } CATCH(e) {
-	RETHROW0("I Can't get a Ack step message from slave : %s");
-      }
-      i++;
-      INFO3("Receive Ack step ack from %s (got %d of %d)",
-	    gras_socket_peer_name(from),
-	    i, SLAVE_COUNT);
-    }
-  }
-  /*********************************  gather ***************************************/
-
-  ask_result=0;
-  for( i=1;i< argc;i++){
-    gras_msg_send(socket[i],gras_msgtype_by_name("ask_result"),&ask_result);
-    INFO1("Send (Ask Result) message to %s",gras_socket_peer_name(socket[i]));
-  }
   /* wait for results */
-  for( i=1;i< argc;i++){
-    gras_msg_wait(600,gras_msgtype_by_name("result"),&from,&result);
-    C.data[(result.i-1)*DATA_MATRIX_SIZE+(result.j-1)]=result.value;
+  for( i=0;i< SLAVE_COUNT;i++){
+    gras_msg_wait(6000,gras_msgtype_by_name("result"),&from,&result);
+    xbt_matrix_copy_values(C,result.C,   submatrix_size,submatrix_size,
+			   submatrix_size*result.linepos,
+			   submatrix_size*result.rowpos,
+			   0,0,NULL);
+    xbt_matrix_free(result.C);
   }
   /*    end of gather   */
-  INFO0 ("The Result of Multiplication is :");
-  display(C);
+  if (DATA_MATRIX_SIZE < 50) {
+     INFO0 ("The Result of Multiplication is :");
+     xbt_matrix_dump(C,"C:res",0,xbt_matrix_dump_display_double);
+  } else {
+     INFO1("Matrix size too big (%d>50) to be displayed here",DATA_MATRIX_SIZE);
+  }
 
+  for(i=0; i<SLAVE_COUNT; i++) {
+     gras_socket_close(socket[i]);
+     xbt_host_free(grid[i]);
+  }
+   
+  xbt_matrix_free(A);
+  xbt_matrix_free(B);
+  xbt_matrix_free(C);
+  gras_exit();
   return 0;
 } /* end_of_master */
 
@@ -257,146 +213,152 @@ int slave(int argc,char *argv[]) {
 
   xbt_ex_t e; 
 
-  int step,port,l,result_ack=0; 
-  double bA,bB;
+  int step,l;
+  xbt_matrix_t bA=xbt_matrix_new(submatrix_size,submatrix_size,
+				 sizeof(double),NULL);
+  xbt_matrix_t bB=xbt_matrix_new(submatrix_size,submatrix_size,
+				 sizeof(double),NULL);
 
-  int myrow,mycol;
-  double mydataA,mydataB;
-  double bC=0;
+  int myline,myrow;
+  xbt_matrix_t mydataA,mydataB;
+  xbt_matrix_t bC=xbt_matrix_double_new_zeros(submatrix_size,submatrix_size);
   
-  //  static end_step;
-
   result_t result;
  
-  gras_socket_t from,sock;  /* to receive from server for steps */
+  gras_socket_t from,sock;  /* to exchange data with my neighbor */
+  gras_socket_t master;     /* for the barrier */
 
   /* sockets for brodcast to other slave */
+  gras_socket_t socket_line[PROC_MATRIX_SIZE-1];
   gras_socket_t socket_row[PROC_MATRIX_SIZE-1];
-  gras_socket_t socket_column[PROC_MATRIX_SIZE-1];
-
+  memset(socket_line,0,sizeof(socket_line));
+  memset(socket_row,0,sizeof(socket_row));
+   
   /* Init the GRAS's infrastructure */
-
   gras_init(&argc, argv);
 
-  /* Get arguments and create sockets */
-
-  port=atoi(argv[1]);
-  
   /*  Create my master socket */
-  sock = gras_socket_server(port);
-  INFO2("Launch %s (port=%d)",argv[0],port);
-  gras_os_sleep(1); //wait to start all slaves 
-
+  sock = gras_socket_server(atoi(argv[1]));
   int i;
-  for (i=1;i<PROC_MATRIX_SIZE;i++){
-    socket_row[i-1]=gras_socket_client(argv[i+1],port);
-    socket_column[i-1]=gras_socket_client(argv[i+PROC_MATRIX_SIZE],port);
-  }
 
   /*  Register the known messages */
   register_messages();
 
   /* Recover my initialized Data and My Position*/
-  init_data_t mydata;
-  INFO0("wait for init Data");
+  s_assignment_t assignment;
+  INFO2("Launch %s (port=%d); wait for my enrole message",argv[0],gras_os_myport());
   TRY {
-    gras_msg_wait(600,gras_msgtype_by_name("init_data"),&from,&mydata);
+    gras_msg_wait(600,gras_msgtype_by_name("assignment"),&master,&assignment);
   } CATCH(e) {
-    RETHROW0("I Can't get a init Data message from master : %s");
+    RETHROW0("Can't get my assignment from master : %s");
   }
-  myrow=mydata.myrow;
-  mycol=mydata.mycol;
-  mydataA=mydata.a;
-  mydataB=mydata.b;
-  INFO4("Receive MY POSITION (%d,%d) and MY INIT DATA ( A=%.3g | B=%.3g )",
-	myrow,mycol,mydataA,mydataB);
-  step=1;
-  
-  do {  //repeat until compute Cb
-    step=PROC_MATRIX_SIZE+1;  // just intilization for loop
-	
-    TRY {
-      gras_msg_wait(200,gras_msgtype_by_name("step"),&from,&step);
-    } CATCH(e) {
-      RETHROW0("I Can't get a Next Step message from master : %s");
+  myline  = assignment.linepos;
+  myrow   = assignment.rowpos;
+  mydataA = assignment.A;
+  mydataB = assignment.B;
+
+  INFO2("Receive my pos (%d,%d) and assignment",myline,myrow);
+
+  /* Get my neighborhood from the assignment message (skipping myself) */
+  int j=0;
+  for (i=0,j=0 ; i<PROC_MATRIX_SIZE ; i++){
+    if (strcmp(gras_os_myname(),assignment.line[i]->name)) {
+      socket_line[j]=gras_socket_client(assignment.line[i]->name,
+					assignment.line[i]->port);
+      j++;
     }
-    INFO1("Receive a step message from master: step = %d ",step);
-
-    if (step < PROC_MATRIX_SIZE ){
-      /* a row brodcast */
-      gras_os_sleep(3);  // IL FAUT EXPRIMER LE TEMPS D'ATTENTE EN FONCTION DE "SLAVE_COUNT"
-      if(myrow==step){
-	INFO2("step(%d) = Myrow(%d)",step,myrow);
-	for (l=1;l < PROC_MATRIX_SIZE ;l++){
-	  gras_msg_send(socket_column[l-1], gras_msgtype_by_name("dataB"), &mydataB);
-	  bB=mydataB;
-	  INFO1("send my data B (%.3g) to my (vertical) neighbors",bB);  
-	}
-      }
-      if(myrow != step){ 
-	INFO2("step(%d) <> Myrow(%d)",step,myrow);
-	TRY {
-	  gras_msg_wait(600,gras_msgtype_by_name("dataB"),
-			&from,&bB);
-	} CATCH(e) {
-	  RETHROW0("I Can't get a data message from row : %s");
-	}
-	INFO2("Receive data B (%.3g) from my neighbor: %s",bB,gras_socket_peer_name(from));
-      }
-      /* a column brodcast */
-      if(mycol==step){
-	for (l=1;l < PROC_MATRIX_SIZE ;l++){
-	  gras_msg_send(socket_row[l-1],gras_msgtype_by_name("dataA"), &mydataA);
-	  bA=mydataA;
-	  INFO1("send my data A (%.3g) to my (horizontal) neighbors",bA);
-	}
-      }
-
-      if(mycol != step){
-	TRY {
-	  gras_msg_wait(1200,gras_msgtype_by_name("dataA"),
-			&from,&bA);
-	} CATCH(e) {
-	  RETHROW0("I Can't get a data message from column : %s");
-	}
-	INFO2("Receive data A (%.3g) from my neighbor : %s ",bA,gras_socket_peer_name(from));
-      }
-      bC+=bA*bB;
-      INFO1(">>>>>>>> My BC = %.3g",bC);
+    xbt_host_free(assignment.line[i]);
+  }
+  for (i=0,j=0 ; i<PROC_MATRIX_SIZE ; i++){
+    if (strcmp(gras_os_myname(),assignment.row[i]->name)) {
+      socket_row[j]=gras_socket_client(assignment.row[i]->name,
+				       assignment.row[i]->port);
+      j++;
+    }
+    xbt_host_free(assignment.row[i]);    
+  }
 
-      /* send a ack msg to master */
+  for (step=0; step<PROC_MATRIX_SIZE;step++) {
 	
-      gras_msg_send(from,gras_msgtype_by_name("step_ack"),&step);
+    /* a line brodcast */
+    if(myline==step){
+       INFO3("LINE: step(%d) = Myline(%d). Broadcast my data (myport=%d).",
+	     step,myline,gras_os_myport());
+       for (l=0;l < PROC_MATRIX_SIZE-1 ;l++)
+	 gras_msg_send(socket_row[l], 
+		       gras_msgtype_by_name("dataB"), 
+		       &mydataB);
 	
-      INFO1("Send ack to master for to end %d th step",step);
+       xbt_matrix_free(bB);
+       bB = xbt_matrix_new_sub(mydataB,
+			       submatrix_size,submatrix_size,
+			       0,0,NULL);	
+    } else {
+      TRY {
+	xbt_matrix_free(bB);
+	gras_msg_wait(600,gras_msgtype_by_name("dataB"),&from,&bB);
+      } CATCH(e) {
+	RETHROW0("Can't get a data message from line : %s");
+      }
+      INFO4("LINE: step(%d) <> Myline(%d). Receive data from %s:%d",step,myline,
+	    gras_socket_peer_name(from), gras_socket_peer_port(from));
     }
-    if(step==PROC_MATRIX_SIZE-1) break;
-	
-  } while (step < PROC_MATRIX_SIZE);
-  /*  wait Message from master to send the result */
- 
-  result.value=bC;
-  result.i=myrow;
-  result.j=mycol;
+
+    /* a row brodcast */
+    if (myrow==step) { 
+       INFO2("ROW: step(%d)=myrow(%d). Broadcast my data",step,myrow);
+       for (l=1;l < PROC_MATRIX_SIZE ;l++)
+	 gras_msg_send(socket_line[l-1],gras_msgtype_by_name("dataA"), &mydataA);
+       xbt_matrix_free(bA);
+       bA = xbt_matrix_new_sub(mydataA,
+			       submatrix_size,submatrix_size,
+			       0,0,NULL);
+    } else {
+      TRY {
+	xbt_matrix_free(bA);
+	gras_msg_wait(1200,gras_msgtype_by_name("dataA"), &from,&bA);
+      } CATCH(e) {
+	RETHROW0("Can't get a data message from row : %s");
+      }
+      INFO3("ROW: step(%d)<>myrow(%d). Receive data from %s",step,myrow,
+	    gras_socket_peer_name(from));
+    }
+    xbt_matrix_double_addmult(bA,bB,bC);
+
+  };
  
+  /* send Result to master */  
+  result.C=bC;
+  result.linepos=myline;
+  result.rowpos=myrow;
+
   TRY {
-    gras_msg_wait(600,gras_msgtype_by_name("ask_result"),
-		  &from,&result_ack);
-  } CATCH(e) {
-    RETHROW0("I Can't get a data message from row : %s");
-  }
-  /* send Result to master */
-  TRY {
-    gras_msg_send(from, gras_msgtype_by_name("result"),&result);
+    gras_msg_send(master, gras_msgtype_by_name("result"),&result);
   } CATCH(e) {
-    // gras_socket_close(from);
     RETHROW0("Failed to send PING to server: %s");
   }
-  INFO3(">>>>>>>> Result: %.3f sent to %s:%d <<<<<<<<",
-	bC,
-	gras_socket_peer_name(from),gras_socket_peer_port(from));
+  INFO2(">>>>>>>> Result sent to %s:%d <<<<<<<<",
+	gras_socket_peer_name(master),gras_socket_peer_port(master));
   /*  Free the allocated resources, and shut GRAS down */
+
+  xbt_matrix_free(bA);
+  xbt_matrix_free(bB);
+  xbt_matrix_free(bC);
+
+  xbt_matrix_free(mydataA);
+  xbt_matrix_free(mydataB);
+  gras_socket_close(sock);
+  gras_socket_close(master);
   gras_socket_close(from);
+  /* FIXME: Some of these sockets are "not known", no idea why *
+  for (l=0; l < PROC_MATRIX_SIZE-1; l++) {
+     if (socket_line[l])
+       gras_socket_close(socket_line[l]);
+     if (socket_row[l])
+       gras_socket_close(socket_row[l]); 
+  }*/
+   
+
   gras_exit();
   INFO0("Done.");
   return 0;