From 037ed0177fab7a21eee2439624cbb4c4db8ccc31 Mon Sep 17 00:00:00 2001 From: Gabriel Corona Date: Tue, 9 Dec 2014 15:29:16 +0100 Subject: [PATCH] [mc] Access memory from another process MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The goal is to be able to move MC in a separate process which should be more robust and easier to develop: * avoid using two heaps (which is combersome); * avoid weird interactions bewteen the MC and the application; * use optimisation for the whole MC process; * avoid the stack-cleaner for the whole MC process. The functions MC_process_read and MC_process_write are defined to abstract memory access: * when the target process is the current processs, they call `memcpy`; * otherwise they call `read` or `write` on `/proc/$pid/mem` (on newer kernels, `process_vm_readv` and `process_vm_writev`) could be used. A lot of bits are missing such as: * access to `std_heap` is currently not process-aware (the current process is used); * access to SIMIX layer from MC; * communcation/synchronisation between the processes; * … Limitations: * for the per-page/chunked snapshot the current implementation uses an extra copy (and one syscall per page), we can do better than this. --- src/mc/mc_checkpoint.c | 25 ++++++---- src/mc/mc_page_snapshot.cpp | 49 +++++++++++++++---- src/mc/mc_page_store.cpp | 1 - src/mc/mc_process.c | 95 +++++++++++++++++++++++++++++++++++-- src/mc/mc_process.h | 31 ++++++++++++ src/mc/mc_snapshot.h | 10 ++-- 6 files changed, 186 insertions(+), 25 deletions(-) diff --git a/src/mc/mc_checkpoint.c b/src/mc/mc_checkpoint.c index 73313227bc..45201cf7af 100644 --- a/src/mc/mc_checkpoint.c +++ b/src/mc/mc_checkpoint.c @@ -121,7 +121,7 @@ static mc_mem_region_t mc_region_new_dense( region->permanent_addr = permanent_addr; region->size = size; region->flat.data = xbt_malloc(size); - memcpy(region->flat.data, permanent_addr, size); + MC_process_read(&mc_model_checker->process, region->flat.data, permanent_addr, size); XBT_DEBUG("New region : type : %d, data : %p (real addr %p), size : %zu", region_type, region->flat.data, permanent_addr, size); return region; @@ -163,11 +163,12 @@ static void MC_region_restore(mc_mem_region_t region, mc_mem_region_t ref_region break; case MC_REGION_STORAGE_TYPE_FLAT: - memcpy(region->permanent_addr, region->flat.data, region->size); + MC_process_write(&mc_model_checker->process, region->flat.data, + region->permanent_addr, region->size); break; case MC_REGION_STORAGE_TYPE_CHUNKED: - mc_region_restore_sparse(region, ref_region); + mc_region_restore_sparse(&mc_model_checker->process, region, ref_region); break; case MC_REGION_STORAGE_TYPE_PRIVATIZED: @@ -690,11 +691,14 @@ mc_snapshot_t MC_take_snapshot(int num_state) MC_get_current_fd(snapshot); + const bool use_soft_dirty = _sg_mc_sparse_checkpoint + && _sg_mc_soft_dirty + && MC_process_is_self(&mc_model_checker->process); + /* Save the std heap and the writable mapped pages of libsimgrid and binary */ MC_get_memory_regions(snapshot); - if (_sg_mc_sparse_checkpoint && _sg_mc_soft_dirty) { + if (use_soft_dirty) mc_softdirty_reset(); - } snapshot->to_ignore = MC_take_snapshot_ignore(); @@ -711,9 +715,8 @@ mc_snapshot_t MC_take_snapshot(int num_state) } MC_snapshot_ignore_restore(snapshot); - if (_sg_mc_sparse_checkpoint && _sg_mc_soft_dirty) { + if (use_soft_dirty) mc_model_checker->parent_snapshot = snapshot; - } return snapshot; } @@ -766,13 +769,17 @@ void MC_restore_snapshot_fds(mc_snapshot_t snapshot) void MC_restore_snapshot(mc_snapshot_t snapshot) { + const bool use_soft_dirty = _sg_mc_sparse_checkpoint + && _sg_mc_soft_dirty + && MC_process_is_self(&mc_model_checker->process); + MC_restore_snapshot_regions(snapshot); MC_restore_snapshot_fds(snapshot); - if (_sg_mc_sparse_checkpoint && _sg_mc_soft_dirty) { + if (use_soft_dirty) { mc_softdirty_reset(); } MC_snapshot_ignore_restore(snapshot); - if (_sg_mc_sparse_checkpoint && _sg_mc_soft_dirty) { + if (use_soft_dirty) { mc_model_checker->parent_snapshot = snapshot; } } diff --git a/src/mc/mc_page_snapshot.cpp b/src/mc/mc_page_snapshot.cpp index dcf0d9bbc8..79caa3af30 100644 --- a/src/mc/mc_page_snapshot.cpp +++ b/src/mc/mc_page_snapshot.cpp @@ -1,3 +1,5 @@ +#include // pread, pwrite + #include "mc_page_store.h" #include "mc_mmu.h" #include "mc_private.h" @@ -20,10 +22,17 @@ extern "C" { * @param reference_pages Snapshot page numbers of the previous soft_dirty_reset (or NULL) * @return Snapshot page numbers of this new snapshot */ -size_t* mc_take_page_snapshot_region(void* data, size_t page_count, uint64_t* pagemap, size_t* reference_pages) +size_t* mc_take_page_snapshot_region(mc_process_t process, + void* data, size_t page_count, uint64_t* pagemap, size_t* reference_pages) { size_t* pagenos = (size_t*) malloc(page_count * sizeof(size_t)); + const bool is_self = MC_process_is_self(process); + + void* temp = NULL; + if (!is_self) + temp = malloc(xbt_pagebits); + for (size_t i=0; i!=page_count; ++i) { bool softclean = pagemap && !(pagemap[i] & SOFT_DIRTY); if (softclean && reference_pages) { @@ -34,10 +43,26 @@ size_t* mc_take_page_snapshot_region(void* data, size_t page_count, uint64_t* pa // Otherwise, we need to store the page the hard way // (by reading its content): void* page = (char*) data + (i << xbt_pagebits); - pagenos[i] = mc_model_checker->pages->store_page(page); + xbt_assert(mc_page_offset(page)==0, "Not at the beginning of a page"); + void* page_data; + if (is_self) { + page_data = page; + } else { + /* Adding another copy (and a syscall) will probably slow things a lot. + TODO, optimize this somehow (at least by grouping the syscalls) + if needed. Either: + - reduce the number of syscalls; + - let the application snapshot itself; + - move the segments in shared memory (this will break `fork` however). + */ + page_data = temp; + MC_process_read(process, temp, page, xbt_pagesize); + } + pagenos[i] = mc_model_checker->pages->store_page(page_data); } } + free(temp); return pagenos; } @@ -59,7 +84,8 @@ void mc_free_page_snapshot_region(size_t* pagenos, size_t page_count) * @param pagemap Linux kernel pagemap values fot this region (or NULL) * @param reference_pages Snapshot page numbers of the previous soft_dirty_reset (or NULL) */ -void mc_restore_page_snapshot_region(void* start_addr, size_t page_count, size_t* pagenos, uint64_t* pagemap, size_t* reference_pagenos) +void mc_restore_page_snapshot_region(mc_process_t process, + void* start_addr, size_t page_count, size_t* pagenos, uint64_t* pagemap, size_t* reference_pagenos) { for (size_t i=0; i!=page_count; ++i) { @@ -73,7 +99,7 @@ void mc_restore_page_snapshot_region(void* start_addr, size_t page_count, size_t // Otherwise, copy the page: void* target_page = mc_page_from_number(start_addr, i); const void* source_page = mc_model_checker->pages->get_page(pagenos[i]); - memcpy(target_page, source_page, xbt_pagesize); + MC_process_write(process, source_page, target_page, xbt_pagesize); } } @@ -163,6 +189,8 @@ mc_mem_region_t mc_region_new_sparse(mc_region_type_t region_type, void *start_addr, void* permanent_addr, size_t size, mc_mem_region_t ref_reg) { + mc_process_t process = &mc_model_checker->process; + mc_mem_region_t region = xbt_new(s_mc_mem_region_t, 1); region->region_type = region_type; region->storage_type = MC_REGION_STORAGE_TYPE_CHUNKED; @@ -177,7 +205,8 @@ mc_mem_region_t mc_region_new_sparse(mc_region_type_t region_type, size_t page_count = mc_page_count(size); uint64_t* pagemap = NULL; - if (_sg_mc_soft_dirty && mc_model_checker->parent_snapshot) { + if (_sg_mc_soft_dirty && mc_model_checker->parent_snapshot && + MC_process_is_self(process)) { pagemap = (uint64_t*) mmalloc_no_memset(mc_heap, sizeof(uint64_t) * page_count); mc_read_pagemap(pagemap, mc_page_number(NULL, permanent_addr), page_count); } @@ -187,7 +216,7 @@ mc_mem_region_t mc_region_new_sparse(mc_region_type_t region_type, reg_page_numbers = ref_reg->chunked.page_numbers; // Take incremental snapshot: - region->chunked.page_numbers = mc_take_page_snapshot_region( + region->chunked.page_numbers = mc_take_page_snapshot_region(process, permanent_addr, page_count, pagemap, reg_page_numbers); if(pagemap) { @@ -196,7 +225,7 @@ mc_mem_region_t mc_region_new_sparse(mc_region_type_t region_type, return region; } -void mc_region_restore_sparse(mc_mem_region_t reg, mc_mem_region_t ref_reg) +void mc_region_restore_sparse(mc_process_t process, mc_mem_region_t reg, mc_mem_region_t ref_reg) { xbt_assert((((uintptr_t)reg->permanent_addr) & (xbt_pagesize-1)) == 0, "Not at the beginning of a page"); @@ -205,7 +234,8 @@ void mc_region_restore_sparse(mc_mem_region_t reg, mc_mem_region_t ref_reg) uint64_t* pagemap = NULL; // Read soft-dirty bits if necessary in order to know which pages have changed: - if (_sg_mc_soft_dirty && mc_model_checker->parent_snapshot) { + if (_sg_mc_soft_dirty && mc_model_checker->parent_snapshot + && MC_process_is_self(process)) { pagemap = (uint64_t*) mmalloc_no_memset(mc_heap, sizeof(uint64_t) * page_count); mc_read_pagemap(pagemap, mc_page_number(NULL, reg->permanent_addr), page_count); } @@ -215,7 +245,8 @@ void mc_region_restore_sparse(mc_mem_region_t reg, mc_mem_region_t ref_reg) if (ref_reg && ref_reg->storage_type == MC_REGION_STORAGE_TYPE_CHUNKED) reg_page_numbers = ref_reg->chunked.page_numbers; - mc_restore_page_snapshot_region(reg->permanent_addr, page_count, reg->chunked.page_numbers, + mc_restore_page_snapshot_region(process, + reg->permanent_addr, page_count, reg->chunked.page_numbers, pagemap, reg_page_numbers); if(pagemap) { diff --git a/src/mc/mc_page_store.cpp b/src/mc/mc_page_store.cpp index 11022f8991..f2c01ee92e 100644 --- a/src/mc/mc_page_store.cpp +++ b/src/mc/mc_page_store.cpp @@ -140,7 +140,6 @@ void s_mc_pages_store::remove_page(size_t pageno) /** Store a page in memory */ size_t s_mc_pages_store::store_page(void* page) { - xbt_assert(mc_page_offset(page)==0, "Not at the beginning of a page"); xbt_assert(top_index_ <= this->capacity_, "top_index is not consistent"); // First, we check if a page with the same content is already in the page diff --git a/src/mc/mc_process.c b/src/mc/mc_process.c index 6b8542ddee..900a6f8d50 100644 --- a/src/mc/mc_process.c +++ b/src/mc/mc_process.c @@ -1,8 +1,10 @@ #include #include #include +#include #include +#include #include #include #include // PROT_* @@ -15,7 +17,9 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(mc_process, mc, "MC process information"); -static void MC_init_memory_map_info(mc_process_t process); +static void MC_process_init_memory_map_info(mc_process_t process); +static void MC_process_open_memory_file(mc_process_t process); + void MC_process_init(mc_process_t process, pid_t pid) { @@ -24,7 +28,9 @@ void MC_process_init(mc_process_t process, pid_t pid) if (pid==getpid()) process->process_flags |= MC_PROCESS_SELF_FLAG; process->memory_map = MC_get_memory_map(pid); - MC_init_memory_map_info(process); + process->memory_file = -1; + MC_process_init_memory_map_info(process); + MC_process_open_memory_file(process); } void MC_process_clear(mc_process_t process) @@ -45,6 +51,9 @@ void MC_process_clear(mc_process_t process) free(process->object_infos); process->object_infos = NULL; process->object_infos_size = 0; + if (process->memory_file >= 0) { + close(process->memory_file); + } } #define SO_RE "\\.so[\\.0-9]*$" @@ -111,7 +120,7 @@ static char* MC_get_lib_name(const char* pathname, struct s_mc_memory_map_re* re } /** @brief Finds the range of the different memory segments and binary paths */ -static void MC_init_memory_map_info(mc_process_t process) +static void MC_process_init_memory_map_info(mc_process_t process) { XBT_INFO("Get debug information ..."); process->maestro_stack_start = NULL; @@ -216,3 +225,83 @@ dw_frame_t MC_process_find_function(mc_process_t process, void *ip) else return MC_file_object_info_find_function(info, ip); } + +// ***** Memory access + +static void MC_process_open_memory_file(mc_process_t process) +{ + if (MC_process_is_self(process) || process->memory_file >= 0) + return; + + const size_t buffer_size = 30; + char buffer[buffer_size]; + int res = snprintf(buffer, buffer_size, "/proc/%lli/mem", (long long) process->pid); + if (res < 0 || res>= buffer_size) { + XBT_ERROR("Could not open memory file descriptor for process %lli", + (long long) process->pid); + return; + } + + int fd = open(buffer, O_RDWR); + if (fd<0) + xbt_die("Could not initialise memory access for remote process"); + process->memory_file = fd; +} + +static ssize_t pread_whole(int fd, void *buf, size_t count, off_t offset) +{ + char* buffer = (char*) buf; + ssize_t real_count = count; + while (count) { + ssize_t res = pread(fd, buffer, count, offset); + if (res >= 0) { + count -= res; + buffer += res; + offset += res; + } else if (res==0) { + return -1; + } else if (errno != EINTR) { + return -1; + } + } + return real_count; +} + +static ssize_t pwrite_whole(int fd, const void *buf, size_t count, off_t offset) +{ + const char* buffer = (const char*) buf; + ssize_t real_count = count; + while (count) { + ssize_t res = pwrite(fd, buffer, count, offset); + if (res >= 0) { + count -= res; + buffer += res; + offset += res; + } else if (res==0) { + return -1; + } else if (errno != EINTR) { + return -1; + } + } + return real_count; +} + +void MC_process_read(mc_process_t process, void* local, const void* remote, size_t len) +{ + if (MC_process_is_self(process)) { + memcpy(local, remote, len); + } else { + if (pread_whole(process->memory_file, local, len, (off_t) remote) < 0) + xbt_die("Read from process %lli failed", (long long) process->pid); + } +} + +void MC_process_write(mc_process_t process, const void* local, void* remote, size_t len) +{ + if (MC_process_is_self(process)) { + memcpy(remote, local, len); + } else { + if (pwrite_whole(process->memory_file, local, len, (off_t) remote) < 0) + xbt_die("Write to process %lli failed", (long long) process->pid); + } +} diff --git a/src/mc/mc_process.h b/src/mc/mc_process.h index 5ea7a48e4e..c8cd39e404 100644 --- a/src/mc/mc_process.h +++ b/src/mc/mc_process.h @@ -7,6 +7,8 @@ #ifndef MC_PROCESS_H #define MC_PROCESS_H +#include + #include "simgrid_config.h" #include @@ -32,11 +34,40 @@ struct s_mc_process { mc_object_info_t binary_info; mc_object_info_t* object_infos; size_t object_infos_size; + int memory_file; }; void MC_process_init(mc_process_t process, pid_t pid); void MC_process_clear(mc_process_t process); +static inline +bool MC_process_is_self(mc_process_t process) +{ + return process->process_flags & MC_PROCESS_SELF_FLAG; +} + +/* Process memory access: */ + +/** Read data from a process memory + * + * @param process the process + * @param local local memory address (destination) + * @param remote target process memory address (source) + * @param len data size + */ +void MC_process_read(mc_process_t process, void* local, const void* remote, size_t len); + +/** Write data to a process memory + * + * @param process the process + * @param local local memory address (source) + * @param remote target process memory address (target) + * @param len data size + */ +void MC_process_write(mc_process_t process, const void* local, void* remote, size_t len); + +/* Functions, variables of the process: */ + mc_object_info_t MC_process_find_object_info(mc_process_t process, void* ip); dw_frame_t MC_process_find_function(mc_process_t process, void* ip); diff --git a/src/mc/mc_snapshot.h b/src/mc/mc_snapshot.h index f31473db24..ca274972a1 100644 --- a/src/mc/mc_snapshot.h +++ b/src/mc/mc_snapshot.h @@ -99,7 +99,7 @@ struct s_mc_mem_region { mc_mem_region_t mc_region_new_sparse(mc_region_type_t type, void *start_addr, void* data_addr, size_t size, mc_mem_region_t ref_reg); void MC_region_destroy(mc_mem_region_t reg); -void mc_region_restore_sparse(mc_mem_region_t reg, mc_mem_region_t ref_reg); +void mc_region_restore_sparse(mc_process_t process, mc_mem_region_t reg, mc_mem_region_t ref_reg); static inline __attribute__ ((always_inline)) bool mc_region_contain(mc_mem_region_t region, void* p) @@ -275,9 +275,13 @@ void MC_free_snapshot(mc_snapshot_t); int mc_important_snapshot(mc_snapshot_t snapshot); -size_t* mc_take_page_snapshot_region(void* data, size_t page_count, uint64_t* pagemap, size_t* reference_pages); +size_t* mc_take_page_snapshot_region(mc_process_t process, + void* data, size_t page_count, uint64_t* pagemap, size_t* reference_pages); void mc_free_page_snapshot_region(size_t* pagenos, size_t page_count); -void mc_restore_page_snapshot_region(void* start_addr, size_t page_count, size_t* pagenos, uint64_t* pagemap, size_t* reference_pagenos); +void mc_restore_page_snapshot_region( + mc_process_t process, + void* start_addr, size_t page_count, size_t* pagenos, + uint64_t* pagemap, size_t* reference_pagenos); void* mc_snapshot_read_fragmented(void* addr, mc_mem_region_t region, void* target, size_t size); -- 2.20.1