-/* Copyright (c) 2015-2018. The SimGrid Team. All rights reserved. */
+/* Copyright (c) 2015-2020. The SimGrid Team. All rights reserved. */
/* This program is free software; you can redistribute it and/or modify it
* under the terms of the license (GNU LGPL) which comes with this package. */
static void smpi_get_executable_global_size()
{
- char buffer[PATH_MAX];
- char* full_name = realpath(xbt_binary_name, buffer);
- if (full_name == nullptr)
- xbt_die("Could not resolve binary file name");
+ char* buffer = realpath(simgrid::xbt::binary_name.c_str(), nullptr);
+ xbt_assert(buffer != nullptr, "Could not resolve real path of binary file '%s'", simgrid::xbt::binary_name.c_str());
+ std::string full_name = buffer;
+ free(buffer);
std::vector<simgrid::xbt::VmMap> map = simgrid::xbt::get_memory_map(getpid());
for (auto i = map.begin(); i != map.end() ; ++i) {
/* Here we are making the assumption that a suitable empty region
following the rw- area is the end of the data segment. It would
be better to check with the size of the data segment. */
- ++i;
- if (i != map.end() && i->pathname.empty() && (i->prot & PROT_RWX) == PROT_RW &&
- (char*)i->start_addr == smpi_data_exe_start + smpi_data_exe_size) {
- // Only count this region if it was not already present in the initial map.
- auto found = std::find_if(begin(initial_vm_map), end(initial_vm_map),
- [&i](const simgrid::xbt::VmMap& m) { return m.start_addr == i->start_addr; });
- if (found == end(initial_vm_map))
- smpi_data_exe_size = (char*)i->end_addr - smpi_data_exe_start;
+ auto j = i + 1;
+ if (j != map.end() && j->pathname.empty() && (j->prot & PROT_RWX) == PROT_RW &&
+ (char*)j->start_addr == smpi_data_exe_start + smpi_data_exe_size) {
+ // Only count the portion of this region not present in the initial map.
+ auto found = std::find_if(initial_vm_map.begin(), initial_vm_map.end(), [&j](const simgrid::xbt::VmMap& m) {
+ return j->start_addr <= m.start_addr && m.start_addr < j->end_addr;
+ });
+ auto end_addr = (found == initial_vm_map.end() ? j->end_addr : found->start_addr);
+ smpi_data_exe_size = (char*)end_addr - smpi_data_exe_start;
}
return;
}
return dest;
}
#else
-#define asan_safe_memcpy(dest, src, n) memcpy(dest, src, n)
+#define asan_safe_memcpy(dest, src, n) memcpy((dest), (src), (n))
#endif
-/** Map a given SMPI privatization segment (make a SMPI process active) */
-void smpi_switch_data_segment(simgrid::s4u::ActorPtr actor)
+/**
+ * @brief Uses shm_open to get a temporary shm, and returns its file descriptor.
+ */
+int smpi_temp_shm_get()
{
- if (smpi_loaded_page == actor->get_pid()) // no need to switch, we've already loaded the one we want
- return;
+ constexpr unsigned VAL_MASK = 0xffffffffUL;
+ static unsigned prev_val = VAL_MASK;
+ char shmname[32]; // cannot be longer than PSHMNAMLEN = 31 on macOS (shm_open raises ENAMETOOLONG otherwise)
+ int fd;
+
+ for (unsigned i = (prev_val + 1) & VAL_MASK; i != prev_val; i = (i + 1) & VAL_MASK) {
+ snprintf(shmname, sizeof(shmname), "/smpi-buffer-%016x", i);
+ fd = shm_open(shmname, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
+ if (fd != -1 || errno != EEXIST) {
+ prev_val = i;
+ break;
+ }
+ }
+ if (fd < 0) {
+ if (errno == EMFILE) {
+ xbt_die("Impossible to create temporary file for memory mapping: %s\n\
+The shm_open() system call failed with the EMFILE error code (too many files). \n\n\
+This means that you reached the system limits concerning the amount of files per process. \
+This is not a surprise if you are trying to virtualize many processes on top of SMPI. \
+Don't panic -- you should simply increase your system limits and try again. \n\n\
+First, check what your limits are:\n\
+ cat /proc/sys/fs/file-max # Gives you the system-wide limit\n\
+ ulimit -Hn # Gives you the per process hard limit\n\
+ ulimit -Sn # Gives you the per process soft limit\n\
+ cat /proc/self/limits # Displays any per-process limitation (including the one given above)\n\n\
+If one of these values is less than the amount of MPI processes that you try to run, then you got the explanation of this error. \
+Ask the Internet about tutorials on how to increase the files limit such as: https://rtcamp.com/tutorials/linux/increase-open-files-limit/",
+ strerror(errno));
+ }
+ xbt_die("Impossible to create temporary file for memory mapping. shm_open: %s", strerror(errno));
+ }
+ XBT_DEBUG("Got temporary shm %s (fd = %d)", shmname, fd);
+ if (shm_unlink(shmname) < 0)
+ XBT_WARN("Could not early unlink %s. shm_unlink: %s", shmname, strerror(errno));
+ return fd;
+}
- // So the job:
- smpi_really_switch_data_segment(actor);
+/**
+ * @brief Mmap a region of size bytes from temporary shm with file descriptor fd.
+ */
+void* smpi_temp_shm_mmap(int fd, size_t size)
+{
+ struct stat st;
+ if (fstat(fd, &st) != 0)
+ xbt_die("Could not stat fd %d: %s", fd, strerror(errno));
+ if (static_cast<off_t>(size) > st.st_size && ftruncate(fd, static_cast<off_t>(size)) != 0)
+ xbt_die("Could not truncate fd %d to %zu: %s", fd, size, strerror(errno));
+ void* mem = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (mem == MAP_FAILED) {
+ xbt_die("Failed to map fd %d with size %zu: %s\n"
+ "If you are running a lot of ranks, you may be exceeding the amount of mappings allowed per process.\n"
+ "On Linux systems, change this value with sudo sysctl -w vm.max_map_count=newvalue (default value: 65536)\n"
+ "Please see "
+ "https://simgrid.org/doc/latest/Configuring_SimGrid.html#configuring-the-user-code-virtualization for more "
+ "information.",
+ fd, size, strerror(errno));
+ }
+ return mem;
}
-/** Map a given SMPI privatization segment (make a SMPI process active) even if SMPI thinks it is already active
+/** Map a given SMPI privatization segment (make an SMPI process active)
*
* When doing a state restoration, the state of the restored variables might not be consistent with the state of the
* virtual memory. In this case, we to change the data segment.
*/
-void smpi_really_switch_data_segment(simgrid::s4u::ActorPtr actor)
+void smpi_switch_data_segment(simgrid::s4u::ActorPtr actor)
{
+ if (smpi_loaded_page == actor->get_pid()) // no need to switch, we've already loaded the one we want
+ return;
+
if (smpi_data_exe_size == 0) // no need to switch
return;
#if HAVE_PRIVATIZATION
// FIXME, cross-process support (mmap across process when necessary)
XBT_DEBUG("Switching data frame to the one of process %ld", actor->get_pid());
- simgrid::smpi::ActorExt* process = smpi_process_remote(actor);
+ const simgrid::smpi::ActorExt* process = smpi_process_remote(actor);
int current = process->privatized_region()->file_descriptor;
- void* tmp = mmap(TOPAGE(smpi_data_exe_start), smpi_data_exe_size, PROT_RW, MAP_FIXED | MAP_SHARED, current, 0);
+ const void* tmp = mmap(TOPAGE(smpi_data_exe_start), smpi_data_exe_size, PROT_RW, MAP_FIXED | MAP_SHARED, current, 0);
if (tmp != TOPAGE(smpi_data_exe_start))
xbt_die("Couldn't map the new region (errno %d): %s", errno, strerror(errno));
smpi_loaded_page = actor->get_pid();
#endif
}
-int smpi_is_privatization_file(char* file)
-{
- const std::string buffer_path("/dev/shm/my-buffer-");
- return buffer_path.compare(0, std::string::npos, file, buffer_path.length()) == 0;
-}
-
/**
* @brief Makes a backup of the segment in memory that stores the global variables of a process.
* This backup is then used to initialize the global variables for every single
// Initializes the memory mapping for a single process and returns the privatization region
smpi_privatization_region_t smpi_init_global_memory_segment_process()
{
- int file_descriptor;
- void* address = nullptr;
- char path[24];
- int status;
-
- do {
- snprintf(path, sizeof(path), "/smpi-buffer-%06x", rand() % 0xffffffU);
- file_descriptor = shm_open(path, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
- } while (file_descriptor == -1 && errno == EEXIST);
- if (file_descriptor < 0) {
- if (errno == EMFILE) {
- xbt_die("Impossible to create temporary file for memory mapping: %s\n\
-The open() system call failed with the EMFILE error code (too many files). \n\n\
-This means that you reached the system limits concerning the amount of files per process. \
-This is not a surprise if you are trying to virtualize many processes on top of SMPI. \
-Don't panic -- you should simply increase your system limits and try again. \n\n\
-First, check what your limits are:\n\
- cat /proc/sys/fs/file-max # Gives you the system-wide limit\n\
- ulimit -Hn # Gives you the per process hard limit\n\
- ulimit -Sn # Gives you the per process soft limit\n\
- cat /proc/self/limits # Displays any per-process limitation (including the one given above)\n\n\
-If one of these values is less than the amount of MPI processes that you try to run, then you got the explanation of this error. \
-Ask the Internet about tutorials on how to increase the files limit such as: https://rtcamp.com/tutorials/linux/increase-open-files-limit/",
- strerror(errno));
- }
- xbt_die("Impossible to create temporary file for memory mapping: %s", strerror(errno));
- }
-
- status = ftruncate(file_descriptor, smpi_data_exe_size);
- if (status)
- xbt_die("Impossible to set the size of the temporary file for memory mapping");
-
- /* Ask for a free region */
- address = mmap(nullptr, smpi_data_exe_size, PROT_RW, MAP_SHARED, file_descriptor, 0);
- if (address == MAP_FAILED)
- xbt_die("Couldn't find a free region for memory mapping");
+ int file_descriptor = smpi_temp_shm_get();
- status = shm_unlink(path);
- if (status)
- xbt_die("Impossible to unlink temporary file for memory mapping");
+ // ask for a free region
+ void* address = smpi_temp_shm_mmap(file_descriptor, smpi_data_exe_size);
// initialize the values
asan_safe_memcpy(address, smpi_data_exe_copy, smpi_data_exe_size);
#endif
}
+static std::vector<unsigned char> sendbuffer;
+static std::vector<unsigned char> recvbuffer;
-//allocate a single buffer for all sends, and an other of all reads, for each actor.
-//buffer size is growing if needed
-
-typedef std::tuple</*buffer adress*/ char*, /*buffer size*/ int> buffer_tuple;
-static std::map<aid_t, buffer_tuple> sendbuffer_map;
-static std::map<aid_t, buffer_tuple> recvbuffer_map;
-static void* smpi_get_tmp_buffer(int size, std::map<aid_t, buffer_tuple> buffer_map)
+//allocate a single buffer for all sends, growing it if needed
+unsigned char* smpi_get_tmp_sendbuffer(size_t size)
{
- // Because this kind of process maintain is own list of buffers and call
- // `smpi_free_tmp_buffer(void* buf)` to free them
if (not smpi_process()->replaying())
- return xbt_malloc(size);
-
- // check if the process is registered
- aid_t id = simgrid::s4u::this_actor::get_pid();
- if (not (buffer_map.find(id) == buffer_map.end()))
- {
- // This tuple represents a buffer and his size
- buffer_tuple buffer_tuple(nullptr, 0);
- buffer_map[id] = buffer_tuple;
- }
- if (std::get<1>(buffer_map[id]) < size){
- std::get<0>(buffer_map[id]) = static_cast<char*>(xbt_realloc(std::get<0>(buffer_map[id]), size));
- std::get<1>(buffer_map[id]) = size;
- }
- return std::get<0>(buffer_map[id]);
+ return new unsigned char[size];
+ // FIXME: a resize() may invalidate a previous pointer. Maybe we need to handle a queue of buffers with a reference
+ // counter. The same holds for smpi_get_tmp_recvbuffer.
+ if (sendbuffer.size() < size)
+ sendbuffer.resize(size);
+ return sendbuffer.data();
}
-void* smpi_get_tmp_sendbuffer(int size)
+//allocate a single buffer for all recv
+unsigned char* smpi_get_tmp_recvbuffer(size_t size)
{
- return smpi_get_tmp_buffer(size, sendbuffer_map);
+ if (not smpi_process()->replaying())
+ return new unsigned char[size];
+ if (recvbuffer.size() < size)
+ recvbuffer.resize(size);
+ return recvbuffer.data();
}
-void* smpi_get_tmp_recvbuffer(int size)
+void smpi_free_tmp_buffer(const unsigned char* buf)
{
- return smpi_get_tmp_buffer(size, recvbuffer_map);
-}
-
-void smpi_free_tmp_buffer(void* buf){
if (not smpi_process()->replaying())
- xbt_free(buf);
+ delete[] buf;
}
-void smpi_free_replay_tmp_buffers(){
- aid_t id = simgrid::s4u::this_actor::get_pid();
- xbt_free(std::get<0>(recvbuffer_map[id]));
- xbt_free(std::get<0>(sendbuffer_map[id]));
+void smpi_free_replay_tmp_buffers()
+{
+ std::vector<unsigned char>().swap(sendbuffer);
+ std::vector<unsigned char>().swap(recvbuffer);
}
