diff options
Diffstat (limited to 'mjit.c')
| -rw-r--r-- | mjit.c | 1073 |
1 files changed, 0 insertions, 1073 deletions
diff --git a/mjit.c b/mjit.c deleted file mode 100644 index 414088e9e4..0000000000 --- a/mjit.c +++ /dev/null @@ -1,1073 +0,0 @@ -/********************************************************************** - - mjit.c - MRI method JIT compiler functions for Ruby's main thread - - Copyright (C) 2017 Vladimir Makarov <vmakarov@redhat.com>. - -**********************************************************************/ - -// Functions in this file are never executed on MJIT worker thread. -// So you can safely use Ruby methods and GC in this file. - -// To share variables privately, include mjit_worker.c instead of linking. - -#include "ruby/internal/config.h" // defines USE_MJIT - -#if USE_MJIT - -#include "constant.h" -#include "id_table.h" -#include "internal.h" -#include "internal/class.h" -#include "internal/cmdlineopt.h" -#include "internal/cont.h" -#include "internal/file.h" -#include "internal/hash.h" -#include "internal/warnings.h" -#include "vm_sync.h" - -#include "mjit_worker.c" - -extern int rb_thread_create_mjit_thread(void (*worker_func)(void)); - -// Return an unique file name in /tmp with PREFIX and SUFFIX and -// number ID. Use getpid if ID == 0. The return file name exists -// until the next function call. -static char * -get_uniq_filename(unsigned long id, const char *prefix, const char *suffix) -{ - char buff[70], *str = buff; - int size = sprint_uniq_filename(buff, sizeof(buff), id, prefix, suffix); - str = 0; - ++size; - str = xmalloc(size); - if (size <= (int)sizeof(buff)) { - memcpy(str, buff, size); - } - else { - sprint_uniq_filename(str, size, id, prefix, suffix); - } - return str; -} - -// Wait until workers don't compile any iseq. It is called at the -// start of GC. -void -mjit_gc_start_hook(void) -{ - if (!mjit_enabled) - return; - CRITICAL_SECTION_START(4, "mjit_gc_start_hook"); - while (in_jit) { - verbose(4, "Waiting wakeup from a worker for GC"); - rb_native_cond_wait(&mjit_client_wakeup, &mjit_engine_mutex); - verbose(4, "Getting wakeup from a worker for GC"); - } - in_gc++; - CRITICAL_SECTION_FINISH(4, "mjit_gc_start_hook"); -} - -// Send a signal to workers to continue iseq compilations. It is -// called at the end of GC. -void -mjit_gc_exit_hook(void) -{ - if (!mjit_enabled) - return; - CRITICAL_SECTION_START(4, "mjit_gc_exit_hook"); - in_gc--; - RUBY_ASSERT_ALWAYS(in_gc >= 0); - if (!in_gc) { - verbose(4, "Sending wakeup signal to workers after GC"); - rb_native_cond_broadcast(&mjit_gc_wakeup); - } - CRITICAL_SECTION_FINISH(4, "mjit_gc_exit_hook"); -} - -// Prohibit calling JIT-ed code and let existing JIT-ed frames exit before the next insn. -void -mjit_cancel_all(const char *reason) -{ - if (!mjit_enabled) - return; - - mjit_call_p = false; - if (mjit_opts.warnings || mjit_opts.verbose) { - fprintf(stderr, "JIT cancel: Disabled JIT-ed code because %s\n", reason); - } -} - -// Deal with ISeq movement from compactor -void -mjit_update_references(const rb_iseq_t *iseq) -{ - if (!mjit_enabled) - return; - - CRITICAL_SECTION_START(4, "mjit_update_references"); - if (ISEQ_BODY(iseq)->jit_unit) { - ISEQ_BODY(iseq)->jit_unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)ISEQ_BODY(iseq)->jit_unit->iseq); - // We need to invalidate JIT-ed code for the ISeq because it embeds pointer addresses. - // To efficiently do that, we use the same thing as TracePoint and thus everything is cancelled for now. - // See mjit.h and tool/ruby_vm/views/_mjit_compile_insn.erb for how `mjit_call_p` is used. - mjit_cancel_all("GC.compact is used"); // TODO: instead of cancelling all, invalidate only this one and recompile it with some threshold. - } - - // Units in stale_units (list of over-speculated and invalidated code) are not referenced from - // `ISEQ_BODY(iseq)->jit_unit` anymore (because new one replaces that). So we need to check them too. - // TODO: we should be able to reduce the number of units checked here. - struct rb_mjit_unit *unit = NULL; - ccan_list_for_each(&stale_units.head, unit, unode) { - if (unit->iseq == iseq) { - unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)unit->iseq); - } - } - CRITICAL_SECTION_FINISH(4, "mjit_update_references"); -} - -// Iseqs can be garbage collected. This function should call when it -// happens. It removes iseq from the unit. -void -mjit_free_iseq(const rb_iseq_t *iseq) -{ - if (!mjit_enabled) - return; - - CRITICAL_SECTION_START(4, "mjit_free_iseq"); - RUBY_ASSERT_ALWAYS(in_gc); - RUBY_ASSERT_ALWAYS(!in_jit); - if (ISEQ_BODY(iseq)->jit_unit) { - // jit_unit is not freed here because it may be referred by multiple - // lists of units. `get_from_list` and `mjit_finish` do the job. - ISEQ_BODY(iseq)->jit_unit->iseq = NULL; - } - // Units in stale_units (list of over-speculated and invalidated code) are not referenced from - // `ISEQ_BODY(iseq)->jit_unit` anymore (because new one replaces that). So we need to check them too. - // TODO: we should be able to reduce the number of units checked here. - struct rb_mjit_unit *unit = NULL; - ccan_list_for_each(&stale_units.head, unit, unode) { - if (unit->iseq == iseq) { - unit->iseq = NULL; - } - } - CRITICAL_SECTION_FINISH(4, "mjit_free_iseq"); -} - -// Free unit list. This should be called only when worker is finished -// because node of unit_queue and one of active_units may have the same unit -// during proceeding unit. -static void -free_list(struct rb_mjit_unit_list *list, bool close_handle_p) -{ - struct rb_mjit_unit *unit = 0, *next; - - ccan_list_for_each_safe(&list->head, unit, next, unode) { - ccan_list_del(&unit->unode); - if (!close_handle_p) unit->handle = NULL; /* Skip dlclose in free_unit() */ - - if (list == &stale_units) { // `free_unit(unit)` crashes after GC.compact on `stale_units` - /* - * TODO: REVERT THIS BRANCH - * Debug the crash on stale_units w/ GC.compact and just use `free_unit(unit)`!! - */ - if (unit->handle && dlclose(unit->handle)) { - mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror()); - } - clean_temp_files(unit); - free(unit); - } - else { - free_unit(unit); - } - } - list->length = 0; -} - -// Register a new continuation with execution context `ec`. Return MJIT info about -// the continuation. -struct mjit_cont * -mjit_cont_new(rb_execution_context_t *ec) -{ - struct mjit_cont *cont; - - // We need to use calloc instead of something like ZALLOC to avoid triggering GC here. - // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup, - // the thread is still being prepared and marking it causes SEGV. - cont = calloc(1, sizeof(struct mjit_cont)); - if (cont == NULL) - rb_memerror(); - cont->ec = ec; - - CRITICAL_SECTION_START(3, "in mjit_cont_new"); - if (first_cont == NULL) { - cont->next = cont->prev = NULL; - } - else { - cont->prev = NULL; - cont->next = first_cont; - first_cont->prev = cont; - } - first_cont = cont; - CRITICAL_SECTION_FINISH(3, "in mjit_cont_new"); - - return cont; -} - -// Unregister continuation `cont`. -void -mjit_cont_free(struct mjit_cont *cont) -{ - CRITICAL_SECTION_START(3, "in mjit_cont_new"); - if (cont == first_cont) { - first_cont = cont->next; - if (first_cont != NULL) - first_cont->prev = NULL; - } - else { - cont->prev->next = cont->next; - if (cont->next != NULL) - cont->next->prev = cont->prev; - } - CRITICAL_SECTION_FINISH(3, "in mjit_cont_new"); - - free(cont); -} - -// Finish work with continuation info. -static void -finish_conts(void) -{ - struct mjit_cont *cont, *next; - - for (cont = first_cont; cont != NULL; cont = next) { - next = cont->next; - xfree(cont); - } -} - -// Create unit for `iseq`. This function may be called from an MJIT worker. -static void -create_unit(const rb_iseq_t *iseq) -{ - struct rb_mjit_unit *unit; - - unit = calloc(1, sizeof(struct rb_mjit_unit)); - if (unit == NULL) - return; - - unit->id = current_unit_num++; - unit->iseq = (rb_iseq_t *)iseq; - ISEQ_BODY(iseq)->jit_unit = unit; -} - -// Return true if given ISeq body should be compiled by MJIT -static inline int -mjit_target_iseq_p(struct rb_iseq_constant_body *body) -{ - return (body->type == ISEQ_TYPE_METHOD || body->type == ISEQ_TYPE_BLOCK) - && !body->builtin_inline_p; -} - -// If recompile_p is true, the call is initiated by mjit_recompile. -// This assumes the caller holds CRITICAL_SECTION when recompile_p is true. -static void -mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info, bool recompile_p) -{ - if (!mjit_enabled || pch_status == PCH_FAILED) - return; - if (!mjit_target_iseq_p(ISEQ_BODY(iseq))) { - ISEQ_BODY(iseq)->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC; // skip mjit_wait - return; - } - - if (!recompile_p) { - CRITICAL_SECTION_START(3, "in add_iseq_to_process"); - - // This prevents multiple Ractors from enqueueing the same ISeq twice. - if (rb_multi_ractor_p() && (uintptr_t)ISEQ_BODY(iseq)->jit_func != NOT_ADDED_JIT_ISEQ_FUNC) { - CRITICAL_SECTION_FINISH(3, "in add_iseq_to_process"); - return; - } - } - - RB_DEBUG_COUNTER_INC(mjit_add_iseq_to_process); - ISEQ_BODY(iseq)->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC; - create_unit(iseq); - if (ISEQ_BODY(iseq)->jit_unit == NULL) - // Failure in creating the unit. - return; - if (compile_info != NULL) - ISEQ_BODY(iseq)->jit_unit->compile_info = *compile_info; - add_to_list(ISEQ_BODY(iseq)->jit_unit, &unit_queue); - if (active_units.length >= mjit_opts.max_cache_size) { - unload_requests++; - } - - if (!recompile_p) { - verbose(3, "Sending wakeup signal to workers in mjit_add_iseq_to_process"); - rb_native_cond_broadcast(&mjit_worker_wakeup); - CRITICAL_SECTION_FINISH(3, "in add_iseq_to_process"); - } -} - -// Add ISEQ to be JITed in parallel with the current thread. -// Unload some JIT codes if there are too many of them. -void -rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq) -{ - mjit_add_iseq_to_process(iseq, NULL, false); -} - -// For this timeout seconds, --jit-wait will wait for JIT compilation finish. -#define MJIT_WAIT_TIMEOUT_SECONDS 60 - -static void -mjit_wait(struct rb_iseq_constant_body *body) -{ - struct timeval tv; - int tries = 0; - tv.tv_sec = 0; - tv.tv_usec = 1000; - while (body->jit_func == (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC) { - tries++; - if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS || pch_status == PCH_FAILED) { - CRITICAL_SECTION_START(3, "in rb_mjit_wait_call to set jit_func"); - body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC; // JIT worker seems dead. Give up. - CRITICAL_SECTION_FINISH(3, "in rb_mjit_wait_call to set jit_func"); - mjit_warning("timed out to wait for JIT finish"); - break; - } - - CRITICAL_SECTION_START(3, "in rb_mjit_wait_call for a client wakeup"); - rb_native_cond_broadcast(&mjit_worker_wakeup); - CRITICAL_SECTION_FINISH(3, "in rb_mjit_wait_call for a client wakeup"); - rb_thread_wait_for(tv); - } -} - -// Wait for JIT compilation finish for --jit-wait, and call the function pointer -// if the compiled result is not NOT_COMPILED_JIT_ISEQ_FUNC. -VALUE -rb_mjit_wait_call(rb_execution_context_t *ec, struct rb_iseq_constant_body *body) -{ - if (worker_stopped) - return Qundef; - - mjit_wait(body); - if ((uintptr_t)body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC) { - return Qundef; - } - return body->jit_func(ec, ec->cfp); -} - -struct rb_mjit_compile_info* -rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body) -{ - assert(body->jit_unit != NULL); - return &body->jit_unit->compile_info; -} - -static void -mjit_recompile(const rb_iseq_t *iseq) -{ - if ((uintptr_t)ISEQ_BODY(iseq)->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC) - return; - - verbose(1, "JIT recompile: %s@%s:%d", RSTRING_PTR(ISEQ_BODY(iseq)->location.label), - RSTRING_PTR(rb_iseq_path(iseq)), FIX2INT(ISEQ_BODY(iseq)->location.first_lineno)); - assert(ISEQ_BODY(iseq)->jit_unit != NULL); - - if (UNLIKELY(mjit_opts.wait)) { - CRITICAL_SECTION_START(3, "in rb_mjit_recompile_iseq"); - remove_from_list(ISEQ_BODY(iseq)->jit_unit, &active_units); - add_to_list(ISEQ_BODY(iseq)->jit_unit, &stale_units); - mjit_add_iseq_to_process(iseq, &ISEQ_BODY(iseq)->jit_unit->compile_info, true); - CRITICAL_SECTION_FINISH(3, "in rb_mjit_recompile_iseq"); - mjit_wait(ISEQ_BODY(iseq)); - } - else { - // Lazily move active_units to stale_units to avoid race conditions around active_units with compaction. - // Also, it's lazily moved to unit_queue as well because otherwise it won't be added to stale_units properly. - // It's good to avoid a race condition between mjit_add_iseq_to_process and mjit_compile around jit_unit as well. - CRITICAL_SECTION_START(3, "in rb_mjit_recompile_iseq"); - ISEQ_BODY(iseq)->jit_unit->stale_p = true; - ISEQ_BODY(iseq)->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC; - pending_stale_p = true; - CRITICAL_SECTION_FINISH(3, "in rb_mjit_recompile_iseq"); - } -} - -// Recompile iseq, disabling send optimization -void -rb_mjit_recompile_send(const rb_iseq_t *iseq) -{ - rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_send_cache = true; - mjit_recompile(iseq); -} - -// Recompile iseq, disabling ivar optimization -void -rb_mjit_recompile_ivar(const rb_iseq_t *iseq) -{ - rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_ivar_cache = true; - mjit_recompile(iseq); -} - -// Recompile iseq, disabling exivar optimization -void -rb_mjit_recompile_exivar(const rb_iseq_t *iseq) -{ - rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_exivar_cache = true; - mjit_recompile(iseq); -} - -// Recompile iseq, disabling method inlining -void -rb_mjit_recompile_inlining(const rb_iseq_t *iseq) -{ - rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_inlining = true; - mjit_recompile(iseq); -} - -// Recompile iseq, disabling getconstant inlining -void -rb_mjit_recompile_const(const rb_iseq_t *iseq) -{ - rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_const_cache = true; - mjit_recompile(iseq); -} - -extern VALUE ruby_archlibdir_path, ruby_prefix_path; - -// Initialize header_file, pch_file, libruby_pathflag. Return true on success. -static bool -init_header_filename(void) -{ - int fd; -#ifdef LOAD_RELATIVE - // Root path of the running ruby process. Equal to RbConfig::TOPDIR. - VALUE basedir_val; -#endif - const char *basedir = ""; - size_t baselen = 0; - char *p; -#ifdef _WIN32 - static const char libpathflag[] = -# ifdef _MSC_VER - "-LIBPATH:" -# else - "-L" -# endif - ; - const size_t libpathflag_len = sizeof(libpathflag) - 1; -#endif - -#ifdef LOAD_RELATIVE - basedir_val = ruby_prefix_path; - basedir = StringValuePtr(basedir_val); - baselen = RSTRING_LEN(basedir_val); -#else - if (getenv("MJIT_SEARCH_BUILD_DIR")) { - // This path is not intended to be used on production, but using build directory's - // header file here because people want to run `make test-all` without running - // `make install`. Don't use $MJIT_SEARCH_BUILD_DIR except for test-all. - - struct stat st; - const char *hdr = dlsym(RTLD_DEFAULT, "MJIT_HEADER"); - if (!hdr) { - verbose(1, "No MJIT_HEADER"); - } - else if (hdr[0] != '/') { - verbose(1, "Non-absolute header file path: %s", hdr); - } - else if (stat(hdr, &st) || !S_ISREG(st.st_mode)) { - verbose(1, "Non-file header file path: %s", hdr); - } - else if ((st.st_uid != getuid()) || (st.st_mode & 022) || - !rb_path_check(hdr)) { - verbose(1, "Unsafe header file: uid=%ld mode=%#o %s", - (long)st.st_uid, (unsigned)st.st_mode, hdr); - return FALSE; - } - else { - // Do not pass PRELOADENV to child processes, on - // multi-arch environment - verbose(3, "PRELOADENV("PRELOADENV")=%s", getenv(PRELOADENV)); - // assume no other PRELOADENV in test-all - unsetenv(PRELOADENV); - verbose(3, "MJIT_HEADER: %s", hdr); - header_file = ruby_strdup(hdr); - if (!header_file) return false; - } - } - else -#endif -#ifndef _MSC_VER - { - // A name of the header file included in any C file generated by MJIT for iseqs. - static const char header_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_MIN_HEADER_NAME; - const size_t header_name_len = sizeof(header_name) - 1; - - header_file = xmalloc(baselen + header_name_len + 1); - p = append_str2(header_file, basedir, baselen); - p = append_str2(p, header_name, header_name_len + 1); - - if ((fd = rb_cloexec_open(header_file, O_RDONLY, 0)) < 0) { - verbose(1, "Cannot access header file: %s", header_file); - xfree(header_file); - header_file = NULL; - return false; - } - (void)close(fd); - } - - pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX "h", ".h.gch"); -#else - { - static const char pch_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_PRECOMPILED_HEADER_NAME; - const size_t pch_name_len = sizeof(pch_name) - 1; - - pch_file = xmalloc(baselen + pch_name_len + 1); - p = append_str2(pch_file, basedir, baselen); - p = append_str2(p, pch_name, pch_name_len + 1); - if ((fd = rb_cloexec_open(pch_file, O_RDONLY, 0)) < 0) { - verbose(1, "Cannot access precompiled header file: %s", pch_file); - xfree(pch_file); - pch_file = NULL; - return false; - } - (void)close(fd); - } -#endif - -#ifdef _WIN32 - basedir_val = ruby_archlibdir_path; - basedir = StringValuePtr(basedir_val); - baselen = RSTRING_LEN(basedir_val); - libruby_pathflag = p = xmalloc(libpathflag_len + baselen + 1); - p = append_str(p, libpathflag); - p = append_str2(p, basedir, baselen); - *p = '\0'; -#endif - - return true; -} - -#ifdef _WIN32 -UINT rb_w32_system_tmpdir(WCHAR *path, UINT len); -#endif - -static char * -system_default_tmpdir(void) -{ - // c.f. ext/etc/etc.c:etc_systmpdir() -#ifdef _WIN32 - WCHAR tmppath[_MAX_PATH]; - UINT len = rb_w32_system_tmpdir(tmppath, numberof(tmppath)); - if (len) { - int blen = WideCharToMultiByte(CP_UTF8, 0, tmppath, len, NULL, 0, NULL, NULL); - char *tmpdir = xmalloc(blen + 1); - WideCharToMultiByte(CP_UTF8, 0, tmppath, len, tmpdir, blen, NULL, NULL); - tmpdir[blen] = '\0'; - return tmpdir; - } -#elif defined _CS_DARWIN_USER_TEMP_DIR - char path[MAXPATHLEN]; - size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path, sizeof(path)); - if (len > 0) { - char *tmpdir = xmalloc(len); - if (len > sizeof(path)) { - confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len); - } - else { - memcpy(tmpdir, path, len); - } - return tmpdir; - } -#endif - return 0; -} - -static int -check_tmpdir(const char *dir) -{ - struct stat st; - - if (!dir) return FALSE; - if (stat(dir, &st)) return FALSE; -#ifndef S_ISDIR -# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#endif - if (!S_ISDIR(st.st_mode)) return FALSE; -#ifndef _WIN32 -# ifndef S_IWOTH -# define S_IWOTH 002 -# endif - if (st.st_mode & S_IWOTH) { -# ifdef S_ISVTX - if (!(st.st_mode & S_ISVTX)) return FALSE; -# else - return FALSE; -# endif - } - if (access(dir, W_OK)) return FALSE; -#endif - return TRUE; -} - -static char * -system_tmpdir(void) -{ - char *tmpdir; -# define RETURN_ENV(name) \ - if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir) - RETURN_ENV("TMPDIR"); - RETURN_ENV("TMP"); - tmpdir = system_default_tmpdir(); - if (check_tmpdir(tmpdir)) return tmpdir; - return ruby_strdup("/tmp"); -# undef RETURN_ENV -} - -// Minimum value for JIT cache size. -#define MIN_CACHE_SIZE 10 -// Default permitted number of units with a JIT code kept in memory. -#define DEFAULT_MAX_CACHE_SIZE 10000 -// A default threshold used to add iseq to JIT. -#define DEFAULT_MIN_CALLS_TO_ADD 10000 - -// Start MJIT worker. Return TRUE if worker is successfully started. -static bool -start_worker(void) -{ - stop_worker_p = false; - worker_stopped = false; - - if (!rb_thread_create_mjit_thread(mjit_worker)) { - mjit_enabled = false; - rb_native_mutex_destroy(&mjit_engine_mutex); - rb_native_cond_destroy(&mjit_pch_wakeup); - rb_native_cond_destroy(&mjit_client_wakeup); - rb_native_cond_destroy(&mjit_worker_wakeup); - rb_native_cond_destroy(&mjit_gc_wakeup); - verbose(1, "Failure in MJIT thread initialization\n"); - return false; - } - return true; -} - -// There's no strndup on Windows -static char* -ruby_strndup(const char *str, size_t n) -{ - char *ret = xmalloc(n + 1); - memcpy(ret, str, n); - ret[n] = '\0'; - return ret; -} - -// Convert "foo bar" to {"foo", "bar", NULL} array. Caller is responsible for -// freeing a returned buffer and its elements. -static char ** -split_flags(const char *flags) -{ - char *buf[MAXPATHLEN]; - int i = 0; - char *next; - for (; flags != NULL; flags = next) { - next = strchr(flags, ' '); - if (next == NULL) { - if (strlen(flags) > 0) - buf[i++] = strdup(flags); - } - else { - if (next > flags) - buf[i++] = ruby_strndup(flags, next - flags); - next++; // skip space - } - } - - char **ret = xmalloc(sizeof(char *) * (i + 1)); - memcpy(ret, buf, sizeof(char *) * i); - ret[i] = NULL; - return ret; -} - -#define opt_match_noarg(s, l, name) \ - opt_match(s, l, name) && (*(s) ? (rb_warn("argument to --mjit-" name " is ignored"), 1) : 1) -#define opt_match_arg(s, l, name) \ - opt_match(s, l, name) && (*(s) ? 1 : (rb_raise(rb_eRuntimeError, "--mjit-" name " needs an argument"), 0)) - -void -mjit_setup_options(const char *s, struct mjit_options *mjit_opt) -{ - const size_t l = strlen(s); - if (l == 0) { - return; - } - else if (opt_match_noarg(s, l, "warnings")) { - mjit_opt->warnings = 1; - } - else if (opt_match(s, l, "debug")) { - if (*s) - mjit_opt->debug_flags = strdup(s + 1); - else - mjit_opt->debug = 1; - } - else if (opt_match_noarg(s, l, "wait")) { - mjit_opt->wait = 1; - } - else if (opt_match_noarg(s, l, "save-temps")) { - mjit_opt->save_temps = 1; - } - else if (opt_match(s, l, "verbose")) { - mjit_opt->verbose = *s ? atoi(s + 1) : 1; - } - else if (opt_match_arg(s, l, "max-cache")) { - mjit_opt->max_cache_size = atoi(s + 1); - } - else if (opt_match_arg(s, l, "min-calls")) { - mjit_opt->min_calls = atoi(s + 1); - } - else { - rb_raise(rb_eRuntimeError, - "invalid MJIT option `%s' (--help will show valid MJIT options)", s); - } -} - -#define M(shortopt, longopt, desc) RUBY_OPT_MESSAGE(shortopt, longopt, desc) -const struct ruby_opt_message mjit_option_messages[] = { - M("--mjit-warnings", "", "Enable printing JIT warnings"), - M("--mjit-debug", "", "Enable JIT debugging (very slow), or add cflags if specified"), - M("--mjit-wait", "", "Wait until JIT compilation finishes every time (for testing)"), - M("--mjit-save-temps", "", "Save JIT temporary files in $TMP or /tmp (for testing)"), - M("--mjit-verbose=num", "", "Print JIT logs of level num or less to stderr (default: 0)"), - M("--mjit-max-cache=num", "", "Max number of methods to be JIT-ed in a cache (default: " - STRINGIZE(DEFAULT_MAX_CACHE_SIZE) ")"), - M("--mjit-min-calls=num", "", "Number of calls to trigger JIT (for testing, default: " - STRINGIZE(DEFAULT_MIN_CALLS_TO_ADD) ")"), - {0} -}; -#undef M - -// Initialize MJIT. Start a thread creating the precompiled header and -// processing ISeqs. The function should be called first for using MJIT. -// If everything is successful, MJIT_INIT_P will be TRUE. -void -mjit_init(const struct mjit_options *opts) -{ - mjit_opts = *opts; - mjit_enabled = true; - mjit_call_p = true; - - // Normalize options - if (mjit_opts.min_calls == 0) - mjit_opts.min_calls = DEFAULT_MIN_CALLS_TO_ADD; - if (mjit_opts.max_cache_size <= 0) - mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE; - if (mjit_opts.max_cache_size < MIN_CACHE_SIZE) - mjit_opts.max_cache_size = MIN_CACHE_SIZE; - - // Initialize variables for compilation -#ifdef _MSC_VER - pch_status = PCH_SUCCESS; // has prebuilt precompiled header -#else - pch_status = PCH_NOT_READY; -#endif - cc_path = CC_COMMON_ARGS[0]; - verbose(2, "MJIT: CC defaults to %s", cc_path); - cc_common_args = xmalloc(sizeof(CC_COMMON_ARGS)); - memcpy((void *)cc_common_args, CC_COMMON_ARGS, sizeof(CC_COMMON_ARGS)); - cc_added_args = split_flags(opts->debug_flags); - xfree(opts->debug_flags); -#if MJIT_CFLAGS_PIPE - // eliminate a flag incompatible with `-pipe` - for (size_t i = 0, j = 0; i < sizeof(CC_COMMON_ARGS) / sizeof(char *); i++) { - if (CC_COMMON_ARGS[i] && strncmp("-save-temps", CC_COMMON_ARGS[i], strlen("-save-temps")) == 0) - continue; // skip -save-temps flag - cc_common_args[j] = CC_COMMON_ARGS[i]; - j++; - } -#endif - - tmp_dir = system_tmpdir(); - verbose(2, "MJIT: tmp_dir is %s", tmp_dir); - - if (!init_header_filename()) { - mjit_enabled = false; - verbose(1, "Failure in MJIT header file name initialization\n"); - return; - } - pch_owner_pid = getpid(); - - // Initialize mutex - rb_native_mutex_initialize(&mjit_engine_mutex); - rb_native_cond_initialize(&mjit_pch_wakeup); - rb_native_cond_initialize(&mjit_client_wakeup); - rb_native_cond_initialize(&mjit_worker_wakeup); - rb_native_cond_initialize(&mjit_gc_wakeup); - - // Make sure the saved_ec of the initial thread's root_fiber is scanned by mark_ec_units. - // - // rb_threadptr_root_fiber_setup for the initial thread is called before mjit_init, - // meaning mjit_cont_new is skipped for the root_fiber. Therefore we need to call - // rb_fiber_init_mjit_cont again with mjit_enabled=true to set the root_fiber's mjit_cont. - rb_fiber_init_mjit_cont(GET_EC()->fiber_ptr); - - // Initialize worker thread - start_worker(); -} - -static void -stop_worker(void) -{ - rb_execution_context_t *ec = GET_EC(); - - while (!worker_stopped) { - verbose(3, "Sending cancel signal to worker"); - CRITICAL_SECTION_START(3, "in stop_worker"); - stop_worker_p = true; // Setting this inside loop because RUBY_VM_CHECK_INTS may make this false. - rb_native_cond_broadcast(&mjit_worker_wakeup); - CRITICAL_SECTION_FINISH(3, "in stop_worker"); - RUBY_VM_CHECK_INTS(ec); - } -} - -// Stop JIT-compiling methods but compiled code is kept available. -VALUE -mjit_pause(bool wait_p) -{ - if (!mjit_enabled) { - rb_raise(rb_eRuntimeError, "MJIT is not enabled"); - } - if (worker_stopped) { - return Qfalse; - } - - // Flush all queued units with no option or `wait: true` - if (wait_p) { - struct timeval tv; - tv.tv_sec = 0; - tv.tv_usec = 1000; - - while (unit_queue.length > 0 && active_units.length < mjit_opts.max_cache_size) { // inverse of condition that waits for mjit_worker_wakeup - CRITICAL_SECTION_START(3, "in mjit_pause for a worker wakeup"); - rb_native_cond_broadcast(&mjit_worker_wakeup); - CRITICAL_SECTION_FINISH(3, "in mjit_pause for a worker wakeup"); - rb_thread_wait_for(tv); - } - } - - stop_worker(); - return Qtrue; -} - -// Restart JIT-compiling methods after mjit_pause. -VALUE -mjit_resume(void) -{ - if (!mjit_enabled) { - rb_raise(rb_eRuntimeError, "MJIT is not enabled"); - } - if (!worker_stopped) { - return Qfalse; - } - - if (!start_worker()) { - rb_raise(rb_eRuntimeError, "Failed to resume MJIT worker"); - } - return Qtrue; -} - -// Skip calling `clean_temp_files` for units which currently exist in the list. -static void -skip_cleaning_object_files(struct rb_mjit_unit_list *list) -{ - struct rb_mjit_unit *unit = NULL, *next; - - // No mutex for list, assuming MJIT worker does not exist yet since it's immediately after fork. - ccan_list_for_each_safe(&list->head, unit, next, unode) { -#if defined(_WIN32) // mswin doesn't reach here either. This is for MinGW. - if (unit->so_file) unit->so_file = NULL; -#endif - } -} - -// This is called after fork initiated by Ruby's method to launch MJIT worker thread -// for child Ruby process. -// -// In multi-process Ruby applications, child Ruby processes do most of the jobs. -// Thus we want child Ruby processes to enqueue ISeqs to MJIT worker's queue and -// call the JIT-ed code. -// -// But unfortunately current MJIT-generated code is process-specific. After the fork, -// JIT-ed code created by parent Ruby process cannot be used in child Ruby process -// because the code could rely on inline cache values (ivar's IC, send's CC) which -// may vary between processes after fork or embed some process-specific addresses. -// -// So child Ruby process can't request parent process to JIT an ISeq and use the code. -// Instead of that, MJIT worker thread is created for all child Ruby processes, even -// while child processes would end up with compiling the same ISeqs. -void -mjit_child_after_fork(void) -{ - if (!mjit_enabled) - return; - - /* Let parent process delete the already-compiled object files. - This must be done before starting MJIT worker on child process. */ - skip_cleaning_object_files(&active_units); - - /* MJIT worker thread is not inherited on fork. Start it for this child process. */ - start_worker(); -} - -// Edit 0 to 1 to enable this feature for investigating hot methods -#define MJIT_COUNTER 0 -#if MJIT_COUNTER -static void -mjit_dump_total_calls(void) -{ - struct rb_mjit_unit *unit; - fprintf(stderr, "[MJIT_COUNTER] total_calls of active_units:\n"); - ccan_list_for_each(&active_units.head, unit, unode) { - const rb_iseq_t *iseq = unit->iseq; - fprintf(stderr, "%8ld: %s@%s:%d\n", ISEQ_BODY(iseq)->total_calls, RSTRING_PTR(ISEQ_BODY(iseq)->location.label), - RSTRING_PTR(rb_iseq_path(iseq)), FIX2INT(ISEQ_BODY(iseq)->location.first_lineno)); - } -} -#endif - -// Finish the threads processing units and creating PCH, finalize -// and free MJIT data. It should be called last during MJIT -// life. -// -// If close_handle_p is true, it calls dlclose() for JIT-ed code. So it should be false -// if the code can still be on stack. ...But it means to leak JIT-ed handle forever (FIXME). -void -mjit_finish(bool close_handle_p) -{ - if (!mjit_enabled) - return; - - // Wait for pch finish - verbose(2, "Stopping worker thread"); - CRITICAL_SECTION_START(3, "in mjit_finish to wakeup from pch"); - // As our threads are detached, we could just cancel them. But it - // is a bad idea because OS processes (C compiler) started by - // threads can produce temp files. And even if the temp files are - // removed, the used C compiler still complaint about their - // absence. So wait for a clean finish of the threads. - while (pch_status == PCH_NOT_READY) { - verbose(3, "Waiting wakeup from make_pch"); - rb_native_cond_wait(&mjit_pch_wakeup, &mjit_engine_mutex); - } - CRITICAL_SECTION_FINISH(3, "in mjit_finish to wakeup from pch"); - - // Stop worker - stop_worker(); - - rb_native_mutex_destroy(&mjit_engine_mutex); - rb_native_cond_destroy(&mjit_pch_wakeup); - rb_native_cond_destroy(&mjit_client_wakeup); - rb_native_cond_destroy(&mjit_worker_wakeup); - rb_native_cond_destroy(&mjit_gc_wakeup); - -#if MJIT_COUNTER - mjit_dump_total_calls(); -#endif - -#ifndef _MSC_VER // mswin has prebuilt precompiled header - if (!mjit_opts.save_temps && getpid() == pch_owner_pid) - remove_file(pch_file); - - xfree(header_file); header_file = NULL; -#endif - xfree((void *)cc_common_args); cc_common_args = NULL; - for (char **flag = cc_added_args; *flag != NULL; flag++) - xfree(*flag); - xfree((void *)cc_added_args); cc_added_args = NULL; - xfree(tmp_dir); tmp_dir = NULL; - xfree(pch_file); pch_file = NULL; - - mjit_call_p = false; - free_list(&unit_queue, close_handle_p); - free_list(&active_units, close_handle_p); - free_list(&compact_units, close_handle_p); - free_list(&stale_units, close_handle_p); - finish_conts(); - - mjit_enabled = false; - verbose(1, "Successful MJIT finish"); -} - -// Called by rb_vm_mark(). -// -// Mark an ISeq being compiled to prevent its CCs from being GC-ed, which -// an MJIT worker may concurrently see. -// -// Also mark active_units so that we do not GC ISeq which may still be -// referred to by mjit_recompile() or compact_all_jit_code(). -void -mjit_mark(void) -{ - if (!mjit_enabled) - return; - RUBY_MARK_ENTER("mjit"); - - // We need to release a lock when calling rb_gc_mark to avoid doubly acquiring - // a lock by by mjit_gc_start_hook inside rb_gc_mark. - // - // Because an MJIT worker may modify active_units anytime, we need to convert - // the linked list to an array to safely loop its ISeqs without keeping a lock. - CRITICAL_SECTION_START(4, "mjit_mark"); - int length = 0; - if (compiling_iseqs != NULL) { - while (compiling_iseqs[length]) length++; - } - length += active_units.length; - const rb_iseq_t **iseqs = ALLOCA_N(const rb_iseq_t *, length); - - struct rb_mjit_unit *unit = NULL; - int i = 0; - if (compiling_iseqs != NULL) { - while (compiling_iseqs[i]) { - iseqs[i] = compiling_iseqs[i]; - i++; - } - } - ccan_list_for_each(&active_units.head, unit, unode) { - iseqs[i] = unit->iseq; - i++; - } - assert(i == length); - CRITICAL_SECTION_FINISH(4, "mjit_mark"); - - for (i = 0; i < length; i++) { - if (iseqs[i] == NULL) // ISeq is GC-ed - continue; - rb_gc_mark((VALUE)iseqs[i]); - } - - RUBY_MARK_LEAVE("mjit"); -} - -// Called by rb_iseq_mark() to mark cc_entries captured for MJIT -void -mjit_mark_cc_entries(const struct rb_iseq_constant_body *const body) -{ - const struct rb_callcache **cc_entries; - if (body->jit_unit && (cc_entries = body->jit_unit->cc_entries) != NULL) { - // It must be `body->jit_unit->cc_entries_size` instead of `body->ci_size` to mark children's cc_entries - for (unsigned int i = 0; i < body->jit_unit->cc_entries_size; i++) { - const struct rb_callcache *cc = cc_entries[i]; - if (cc != NULL && vm_cc_markable(cc)) { - // Pin `cc` and `cc->cme` against GC.compact as their addresses may be written in JIT-ed code. - rb_gc_mark((VALUE)cc); - rb_gc_mark((VALUE)vm_cc_cme(cc)); - } - } - } -} - -#endif // USE_MJIT |