diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | common.mk | 34 | ||||
| -rw-r--r-- | compile.c | 1 | ||||
| -rw-r--r-- | mjit.h | 2 | ||||
| -rw-r--r-- | ruby.c | 2 | ||||
| -rw-r--r-- | template/Makefile.in | 2 | ||||
| -rw-r--r-- | tool/ruby_vm/views/ujit_hooks.inc.erb (renamed from tool/ruby_vm/views/ujit_examples.inc.erb) | 0 | ||||
| -rw-r--r-- | ujit.h (renamed from ujit_compile.h) | 28 | ||||
| -rw-r--r-- | ujit_codegen.c (renamed from ujit_compile.c) | 383 | ||||
| -rw-r--r-- | ujit_codegen.h | 10 | ||||
| -rw-r--r-- | ujit_core.c | 68 | ||||
| -rw-r--r-- | ujit_core.h | 50 | ||||
| -rw-r--r-- | ujit_iface.c | 260 | ||||
| -rw-r--r-- | ujit_iface.h | 27 | ||||
| -rw-r--r-- | version.c | 4 | ||||
| -rw-r--r-- | vm_method.c | 2 | ||||
| -rw-r--r-- | win32/Makefile.sub | 2 |
17 files changed, 485 insertions, 392 deletions
diff --git a/.gitignore b/.gitignore index 4c2a8ca452..4fa89e9dfa 100644 --- a/.gitignore +++ b/.gitignore @@ -230,4 +230,4 @@ lcov*.info /include/ruby-*/*/rb_mjit_min_header-*.h # UJIT -/ujit_examples.h +/ujit_hooks.inc @@ -152,7 +152,9 @@ COMMONOBJS = array.$(OBJEXT) \ vm_trace.$(OBJEXT) \ ujit_asm.$(OBJEXT) \ ujit_utils.$(OBJEXT) \ - ujit_compile.$(OBJEXT) \ + ujit_core.$(OBJEXT) \ + ujit_codegen.$(OBJEXT) \ + ujit_iface.$(OBJEXT) \ $(COROUTINE_OBJ) \ $(DTRACE_OBJ) \ $(BUILTIN_ENCOBJS) \ @@ -1108,7 +1110,7 @@ incs: $(INSNS) {$(VPATH)}node_name.inc {$(VPATH)}known_errors.inc \ insns: $(INSNS) -ujit_examples.inc: vm.$(OBJEXT) +ujit_hooks.inc: vm.$(OBJEXT) id.h: $(tooldir)/generic_erb.rb $(srcdir)/template/id.h.tmpl $(srcdir)/defs/id.def $(ECHO) generating $@ @@ -3096,7 +3098,7 @@ compile.$(OBJEXT): {$(VPATH)}st.h compile.$(OBJEXT): {$(VPATH)}subst.h compile.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h compile.$(OBJEXT): {$(VPATH)}thread_native.h -compile.$(OBJEXT): {$(VPATH)}ujit_compile.h +compile.$(OBJEXT): {$(VPATH)}ujit.h compile.$(OBJEXT): {$(VPATH)}util.h compile.$(OBJEXT): {$(VPATH)}vm_callinfo.h compile.$(OBJEXT): {$(VPATH)}vm_core.h @@ -3476,7 +3478,7 @@ cont.$(OBJEXT): {$(VPATH)}st.h cont.$(OBJEXT): {$(VPATH)}subst.h cont.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h cont.$(OBJEXT): {$(VPATH)}thread_native.h -cont.$(OBJEXT): {$(VPATH)}ujit_compile.h +cont.$(OBJEXT): {$(VPATH)}ujit.h cont.$(OBJEXT): {$(VPATH)}vm_core.h cont.$(OBJEXT): {$(VPATH)}vm_debug.h cont.$(OBJEXT): {$(VPATH)}vm_opts.h @@ -5567,7 +5569,7 @@ eval.$(OBJEXT): {$(VPATH)}st.h eval.$(OBJEXT): {$(VPATH)}subst.h eval.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h eval.$(OBJEXT): {$(VPATH)}thread_native.h -eval.$(OBJEXT): {$(VPATH)}ujit_compile.h +eval.$(OBJEXT): {$(VPATH)}ujit.h eval.$(OBJEXT): {$(VPATH)}vm.h eval.$(OBJEXT): {$(VPATH)}vm_core.h eval.$(OBJEXT): {$(VPATH)}vm_debug.h @@ -6016,7 +6018,7 @@ gc.$(OBJEXT): {$(VPATH)}thread.h gc.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h gc.$(OBJEXT): {$(VPATH)}thread_native.h gc.$(OBJEXT): {$(VPATH)}transient_heap.h -gc.$(OBJEXT): {$(VPATH)}ujit_compile.h +gc.$(OBJEXT): {$(VPATH)}ujit.h gc.$(OBJEXT): {$(VPATH)}util.h gc.$(OBJEXT): {$(VPATH)}vm_callinfo.h gc.$(OBJEXT): {$(VPATH)}vm_core.h @@ -7008,7 +7010,7 @@ iseq.$(OBJEXT): {$(VPATH)}subst.h iseq.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h iseq.$(OBJEXT): {$(VPATH)}thread_native.h iseq.$(OBJEXT): {$(VPATH)}ujit_asm.h -iseq.$(OBJEXT): {$(VPATH)}ujit_compile.h +iseq.$(OBJEXT): {$(VPATH)}ujit.h iseq.$(OBJEXT): {$(VPATH)}util.h iseq.$(OBJEXT): {$(VPATH)}vm_callinfo.h iseq.$(OBJEXT): {$(VPATH)}vm_core.h @@ -8706,7 +8708,7 @@ mjit.$(OBJEXT): {$(VPATH)}subst.h mjit.$(OBJEXT): {$(VPATH)}thread.h mjit.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h mjit.$(OBJEXT): {$(VPATH)}thread_native.h -mjit.$(OBJEXT): {$(VPATH)}ujit_compile.h +mjit.$(OBJEXT): {$(VPATH)}ujit.h mjit.$(OBJEXT): {$(VPATH)}util.h mjit.$(OBJEXT): {$(VPATH)}vm_callinfo.h mjit.$(OBJEXT): {$(VPATH)}vm_core.h @@ -8919,7 +8921,7 @@ mjit_compile.$(OBJEXT): {$(VPATH)}st.h mjit_compile.$(OBJEXT): {$(VPATH)}subst.h mjit_compile.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h mjit_compile.$(OBJEXT): {$(VPATH)}thread_native.h -mjit_compile.$(OBJEXT): {$(VPATH)}ujit_compile.h +mjit_compile.$(OBJEXT): {$(VPATH)}ujit.h mjit_compile.$(OBJEXT): {$(VPATH)}vm_callinfo.h mjit_compile.$(OBJEXT): {$(VPATH)}vm_core.h mjit_compile.$(OBJEXT): {$(VPATH)}vm_exec.h @@ -12508,7 +12510,7 @@ ruby.$(OBJEXT): {$(VPATH)}subst.h ruby.$(OBJEXT): {$(VPATH)}thread.h ruby.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h ruby.$(OBJEXT): {$(VPATH)}thread_native.h -ruby.$(OBJEXT): {$(VPATH)}ujit_compile.h +ruby.$(OBJEXT): {$(VPATH)}ujit.h ruby.$(OBJEXT): {$(VPATH)}util.h ruby.$(OBJEXT): {$(VPATH)}vm_core.h ruby.$(OBJEXT): {$(VPATH)}vm_opts.h @@ -14471,7 +14473,7 @@ thread.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h thread.$(OBJEXT): {$(VPATH)}thread_native.h thread.$(OBJEXT): {$(VPATH)}thread_sync.c thread.$(OBJEXT): {$(VPATH)}timev.h -thread.$(OBJEXT): {$(VPATH)}ujit_compile.h +thread.$(OBJEXT): {$(VPATH)}ujit.h thread.$(OBJEXT): {$(VPATH)}vm_core.h thread.$(OBJEXT): {$(VPATH)}vm_debug.h thread.$(OBJEXT): {$(VPATH)}vm_opts.h @@ -15236,8 +15238,8 @@ ujit_compile.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h ujit_compile.$(OBJEXT): {$(VPATH)}thread_native.h ujit_compile.$(OBJEXT): {$(VPATH)}ujit_asm.h ujit_compile.$(OBJEXT): {$(VPATH)}ujit_compile.c -ujit_compile.$(OBJEXT): {$(VPATH)}ujit_compile.h -ujit_compile.$(OBJEXT): {$(VPATH)}ujit_examples.inc +ujit_compile.$(OBJEXT): {$(VPATH)}ujit.h +ujit_compile.$(OBJEXT): {$(VPATH)}ujit_hooks.inc ujit_compile.$(OBJEXT): {$(VPATH)}ujit_utils.h ujit_compile.$(OBJEXT): {$(VPATH)}vm_callinfo.h ujit_compile.$(OBJEXT): {$(VPATH)}vm_core.h @@ -15810,7 +15812,7 @@ version.$(OBJEXT): {$(VPATH)}st.h version.$(OBJEXT): {$(VPATH)}subst.h version.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h version.$(OBJEXT): {$(VPATH)}thread_native.h -version.$(OBJEXT): {$(VPATH)}ujit_compile.h +version.$(OBJEXT): {$(VPATH)}ujit.h version.$(OBJEXT): {$(VPATH)}version.c version.$(OBJEXT): {$(VPATH)}vm_core.h version.$(OBJEXT): {$(VPATH)}vm_opts.h @@ -16043,7 +16045,7 @@ vm.$(OBJEXT): {$(VPATH)}st.h vm.$(OBJEXT): {$(VPATH)}subst.h vm.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h vm.$(OBJEXT): {$(VPATH)}thread_native.h -vm.$(OBJEXT): {$(VPATH)}ujit_compile.h +vm.$(OBJEXT): {$(VPATH)}ujit.h vm.$(OBJEXT): {$(VPATH)}variable.h vm.$(OBJEXT): {$(VPATH)}vm.c vm.$(OBJEXT): {$(VPATH)}vm.h @@ -16849,7 +16851,7 @@ vm_trace.$(OBJEXT): {$(VPATH)}subst.h vm_trace.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h vm_trace.$(OBJEXT): {$(VPATH)}thread_native.h vm_trace.$(OBJEXT): {$(VPATH)}trace_point.rbinc -vm_trace.$(OBJEXT): {$(VPATH)}ujit_compile.h +vm_trace.$(OBJEXT): {$(VPATH)}ujit.h vm_trace.$(OBJEXT): {$(VPATH)}vm_core.h vm_trace.$(OBJEXT): {$(VPATH)}vm_opts.h vm_trace.$(OBJEXT): {$(VPATH)}vm_trace.c @@ -43,7 +43,6 @@ #include "builtin.h" #include "insns.inc" #include "insns_info.inc" -#include "ujit_compile.h" #undef RUBY_UNTYPED_DATA_WARNING #define RUBY_UNTYPED_DATA_WARNING 0 @@ -16,7 +16,7 @@ #include "debug_counter.h" #include "ruby.h" -#include "ujit_compile.h" +#include "ujit.h" // Special address values of a function generated from the // corresponding iseq by MJIT: @@ -59,7 +59,7 @@ #include "internal/process.h" #include "internal/variable.h" #include "mjit.h" -#include "ujit_compile.h" +#include "ujit.h" #include "ruby/encoding.h" #include "ruby/thread.h" #include "ruby/util.h" diff --git a/template/Makefile.in b/template/Makefile.in index 557c9922f2..c71681b4c4 100644 --- a/template/Makefile.in +++ b/template/Makefile.in @@ -590,7 +590,7 @@ update-known-errors: $(IFCHANGE) $(srcdir)/defs/known_errors.def - INSNS = opt_sc.inc optinsn.inc optunifs.inc insns.inc insns_info.inc \ - vmtc.inc vm.inc mjit_compile.inc ujit_examples.inc + vmtc.inc vm.inc mjit_compile.inc ujit_hooks.inc $(INSNS): $(srcdir)/insns.def vm_opts.h \ $(srcdir)/defs/opt_operand.def $(srcdir)/defs/opt_insn_unif.def \ diff --git a/tool/ruby_vm/views/ujit_examples.inc.erb b/tool/ruby_vm/views/ujit_hooks.inc.erb index 9409c996b1..9409c996b1 100644 --- a/tool/ruby_vm/views/ujit_examples.inc.erb +++ b/tool/ruby_vm/views/ujit_hooks.inc.erb diff --git a/ujit_compile.h b/ujit.h index 15078386ff..b4a83ae0e6 100644 --- a/ujit_compile.h +++ b/ujit.h @@ -1,11 +1,31 @@ -#ifndef UJIT_COMPILE_H -#define UJIT_COMPILE_H 1 +// +// This file contains definitions uJIT exposes to the CRuby codebase +// + +#ifndef UJIT_H +#define UJIT_H 1 #include "stddef.h" #include "stdint.h" #include "stdbool.h" #include "method.h" +#ifdef _WIN32 +#define PLATFORM_SUPPORTED_P 0 +#else +#define PLATFORM_SUPPORTED_P 1 +#endif + +#ifndef UJIT_CHECK_MODE +#define UJIT_CHECK_MODE 0 +#endif + +// >= 1: print when output code invalidation happens +// >= 2: dump list of instructions when regions compile +#ifndef UJIT_DUMP_MODE +#define UJIT_DUMP_MODE 0 +#endif + #ifndef rb_iseq_t typedef struct rb_iseq_struct rb_iseq_t; #define rb_iseq_t rb_iseq_t @@ -24,7 +44,7 @@ bool rb_ujit_enabled_p(void) #define UJIT_CALL_THRESHOLD (10u) void rb_ujit_method_lookup_change(VALUE cme_or_cc); -void rb_ujit_init(void); void rb_ujit_compile_iseq(const rb_iseq_t *iseq); +void rb_ujit_init(void); -#endif +#endif // #ifndef UJIT_H diff --git a/ujit_compile.c b/ujit_codegen.c index 773920739c..09a0acd76d 100644 --- a/ujit_compile.c +++ b/ujit_codegen.c @@ -8,59 +8,15 @@ #include "internal/compile.h" #include "internal/class.h" #include "insns_info.inc" -#include "ujit_compile.h" +#include "ujit.h" +#include "ujit_iface.h" +#include "ujit_core.h" +#include "ujit_codegen.h" #include "ujit_asm.h" #include "ujit_utils.h" +#include "ujit_hooks.inc" -// TODO: give ujit_examples.inc some more meaningful file name -// eg ujit_hook.h -#include "ujit_examples.inc" - -#ifdef _WIN32 -#define PLATFORM_SUPPORTED_P 0 -#else -#define PLATFORM_SUPPORTED_P 1 -#endif - -#ifndef UJIT_CHECK_MODE -#define UJIT_CHECK_MODE 0 -#endif - -// >= 1: print when output code invalidation happens -// >= 2: dump list of instructions when regions compile -#ifndef UJIT_DUMP_MODE -#define UJIT_DUMP_MODE 0 -#endif - -bool rb_ujit_enabled; - -// Hash table of encoded instructions -extern st_table *rb_encoded_insn_data; - -// Code generation context -typedef struct ctx_struct -{ - // Current PC - VALUE *pc; - - // Difference between the current stack pointer and actual stack top - int32_t stack_diff; - - // The iseq that owns the region that is compiling - const rb_iseq_t *iseq; - - // Index in the iseq of the opcode we are replacing - size_t start_idx; - - // The start of the generated code - uint8_t *code_ptr; - - // Whether we know self is a heap object - bool self_is_object; - -} ctx_t; - -// MicroJIT code generation function signature +// Code generation function signature typedef bool (*codegen_fn)(codeblock_t* cb, codeblock_t* ocb, ctx_t* ctx); // Map from YARV opcodes to code generation functions @@ -74,243 +30,6 @@ static codeblock_t* cb = NULL; static codeblock_t outline_block; static codeblock_t* ocb = NULL; -// Register MicroJIT receives the CFP and EC into -#define REG_CFP RDI -#define REG_EC RSI - -// Register MicroJIT loads the SP into -#define REG_SP RDX - -// Scratch registers used by MicroJIT -#define REG0 RAX -#define REG1 RCX -#define REG0_32 EAX -#define REG1_32 ECX - -// Keep track of mapping from instructions to generated code -// See comment for rb_encoded_insn_data in iseq.c -static void -addr2insn_bookkeeping(void *code_ptr, int insn) -{ - const void * const *table = rb_vm_get_insns_address_table(); - const void * const translated_address = table[insn]; - st_data_t encoded_insn_data; - if (st_lookup(rb_encoded_insn_data, (st_data_t)translated_address, &encoded_insn_data)) { - st_insert(rb_encoded_insn_data, (st_data_t)code_ptr, encoded_insn_data); - } - else { - rb_bug("ujit: failed to find info for original instruction while dealing with addr2insn"); - } -} - -// GC root for interacting with the GC -struct ujit_root_struct {}; - -// Map cme_or_cc => [[iseq, offset]]. An entry in the map means compiled code at iseq[offset] -// is only valid when cme_or_cc is valid -static st_table *method_lookup_dependency; - -struct compiled_region_array { - int32_t size; - int32_t capa; - struct compiled_region { - const rb_iseq_t *iseq; - size_t start_idx; - uint8_t *code; - } data[]; -}; - -// Add an element to a region array, or allocate a new region array. -static struct compiled_region_array * -add_compiled_region(struct compiled_region_array *array, const rb_iseq_t *iseq, size_t start_idx, uint8_t *code) -{ - if (!array) { - // Allocate a brand new array with space for one - array = malloc(sizeof(*array) + sizeof(struct compiled_region)); - if (!array) { - return NULL; - } - array->size = 0; - array->capa = 1; - } - if (array->size == INT32_MAX) { - return NULL; - } - // Check if the region is already present - for (int32_t i = 0; i < array->size; i++) { - if (array->data[i].iseq == iseq && array->data[i].start_idx == start_idx) { - return array; - } - } - if (array->size + 1 > array->capa) { - // Double the array's capacity. - int64_t double_capa = ((int64_t)array->capa) * 2; - int32_t new_capa = (int32_t)double_capa; - if (new_capa != double_capa) { - return NULL; - } - array = realloc(array, sizeof(*array) + new_capa * sizeof(struct compiled_region)); - if (array == NULL) { - return NULL; - } - array->capa = new_capa; - } - - int32_t size = array->size; - array->data[size].iseq = iseq; - array->data[size].start_idx = start_idx; - array->data[size].code = code; - array->size++; - return array; -} - -static int -add_lookup_dependency_i(st_data_t *key, st_data_t *value, st_data_t data, int existing) -{ - ctx_t *ctx = (ctx_t *)data; - struct compiled_region_array *regions = NULL; - if (existing) { - regions = (struct compiled_region_array *)*value; - } - regions = add_compiled_region(regions, ctx->iseq, ctx->start_idx, ctx->code_ptr); - if (!regions) { - rb_bug("ujit: failed to add method lookup dependency"); // TODO: we could bail out of compiling instead - } - *value = (st_data_t)regions; - return ST_CONTINUE; -} - -// Store info to remember that the currently compiling region is only valid while cme and cc and valid. -static void -ujit_assume_method_lookup_stable(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme, ctx_t *ctx) -{ - st_update(method_lookup_dependency, (st_data_t)cme, add_lookup_dependency_i, (st_data_t)ctx); - st_update(method_lookup_dependency, (st_data_t)cc, add_lookup_dependency_i, (st_data_t)ctx); - // FIXME: This is a leak! When either the cme or the cc become invalid, the other also needs to go -} - -static int -ujit_root_mark_i(st_data_t k, st_data_t v, st_data_t ignore) -{ - // FIXME: This leaks everything that end up in the dependency table! - // One way to deal with this is with weak references... - rb_gc_mark((VALUE)k); - struct compiled_region_array *regions = (void *)v; - for (int32_t i = 0; i < regions->size; i++) { - rb_gc_mark((VALUE)regions->data[i].iseq); - } - - return ST_CONTINUE; -} - -// GC callback during mark phase -static void -ujit_root_mark(void *ptr) -{ - if (method_lookup_dependency) { - st_foreach(method_lookup_dependency, ujit_root_mark_i, 0); - } -} - -static void -ujit_root_free(void *ptr) -{ - // Do nothing. The root lives as long as the process. -} - -static size_t -ujit_root_memsize(const void *ptr) -{ - // Count off-gc-heap allocation size of the dependency table - return st_memsize(method_lookup_dependency); // TODO: more accurate accounting -} - -// Custom type for interacting with the GC -// TODO: compaction support -// TODO: make this write barrier protected -static const rb_data_type_t ujit_root_type = { - "ujit_root", - {ujit_root_mark, ujit_root_free, ujit_root_memsize, }, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY -}; - -static int -opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) -{ - const VALUE at_pc = *pc; - if (FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED)) { - return rb_vm_insn_addr2opcode((const void *)at_pc); - } - else { - return (int)at_pc; - } -} - -// Get the current instruction opcode from the context object -static int -ctx_get_opcode(ctx_t *ctx) -{ - return opcode_at_pc(ctx->iseq, ctx->pc); -} - -// Get an instruction argument from the context object -static VALUE -ctx_get_arg(ctx_t* ctx, size_t arg_idx) -{ - assert (arg_idx + 1 < insn_len(ctx_get_opcode(ctx))); - return *(ctx->pc + arg_idx + 1); -} - -/* -Get an operand for the adjusted stack pointer address -*/ -static x86opnd_t -ctx_sp_opnd(ctx_t* ctx, int32_t offset_bytes) -{ - int32_t offset = (ctx->stack_diff) * 8 + offset_bytes; - return mem_opnd(64, REG_SP, offset); -} - -/* -Make space on the stack for N values -Return a pointer to the new stack top -*/ -static x86opnd_t -ctx_stack_push(ctx_t* ctx, size_t n) -{ - ctx->stack_diff += n; - - // SP points just above the topmost value - int32_t offset = (ctx->stack_diff - 1) * 8; - return mem_opnd(64, REG_SP, offset); -} - -/* -Pop N values off the stack -Return a pointer to the stack top before the pop operation -*/ -static x86opnd_t -ctx_stack_pop(ctx_t* ctx, size_t n) -{ - // SP points just above the topmost value - int32_t offset = (ctx->stack_diff - 1) * 8; - x86opnd_t top = mem_opnd(64, REG_SP, offset); - - ctx->stack_diff -= n; - - return top; -} - -static x86opnd_t -ctx_stack_opnd(ctx_t* ctx, int32_t idx) -{ - // SP points just above the topmost value - int32_t offset = (ctx->stack_diff - 1 - idx) * 8; - x86opnd_t opnd = mem_opnd(64, REG_SP, offset); - - return opnd; -} - // Ruby instruction entry static void ujit_gen_entry(codeblock_t* cb) @@ -382,7 +101,7 @@ Compile a sequence of bytecode instructions starting at `insn_idx`. Return the index to the first instruction not compiled in the sequence through `next_ujit_idx`. Return `NULL` in case compilation fails. */ -static uint8_t * +uint8_t * ujit_compile_insn(const rb_iseq_t *iseq, unsigned int insn_idx, unsigned int *next_ujit_idx) { assert (cb != NULL); @@ -469,7 +188,7 @@ ujit_compile_insn(const rb_iseq_t *iseq, unsigned int insn_idx, unsigned int *ne // Generate code to exit to the interpreter ujit_gen_exit(cb, &ctx, &encoded[*next_ujit_idx]); - addr2insn_bookkeeping(code_ptr, first_opcode); + map_addr2insn(code_ptr, first_opcode); if (UJIT_DUMP_MODE >= 2) { // Dump list of compiled instrutions @@ -486,6 +205,15 @@ ujit_compile_insn(const rb_iseq_t *iseq, unsigned int insn_idx, unsigned int *ne return code_ptr; } + + + + + + + + + static bool gen_dup(codeblock_t* cb, codeblock_t* ocb, ctx_t* ctx) { @@ -1098,7 +826,8 @@ gen_opt_send_without_block(codeblock_t* cb, codeblock_t* ocb, ctx_t* ctx) //print_str(cb, "before C call"); - ujit_assume_method_lookup_stable(cd->cc, cme, ctx); + assume_method_lookup_stable(cd->cc, cme, ctx); + // Call the C function // VALUE ret = (cfunc->func)(recv, argv[0], argv[1]); // cfunc comes from compile-time cme->def, which we assume to be stable. @@ -1134,75 +863,8 @@ gen_opt_send_without_block(codeblock_t* cb, codeblock_t* ocb, ctx_t* ctx) } void -rb_ujit_compile_iseq(const rb_iseq_t *iseq) -{ -#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE - RB_VM_LOCK_ENTER(); - VALUE *encoded = (VALUE *)iseq->body->iseq_encoded; - - unsigned int insn_idx; - unsigned int next_ujit_idx = 0; - - for (insn_idx = 0; insn_idx < iseq->body->iseq_size; /* */) { - int insn = opcode_at_pc(iseq, &encoded[insn_idx]); - int len = insn_len(insn); - - uint8_t *native_code_ptr = NULL; - - // If ujit hasn't already compiled this instruction - if (insn_idx >= next_ujit_idx) { - native_code_ptr = ujit_compile_insn(iseq, insn_idx, &next_ujit_idx); - } - - if (native_code_ptr) { - encoded[insn_idx] = (VALUE)native_code_ptr; - } - insn_idx += len; - } - RB_VM_LOCK_LEAVE(); -#endif -} - -// Callback when cme or cc become invalid -void -rb_ujit_method_lookup_change(VALUE cme_or_cc) -{ - if (!method_lookup_dependency) return; - - RUBY_ASSERT(IMEMO_TYPE_P(cme_or_cc, imemo_ment) || IMEMO_TYPE_P(cme_or_cc, imemo_callcache)); - - st_data_t image; - if (st_lookup(method_lookup_dependency, (st_data_t)cme_or_cc, &image)) { - struct compiled_region_array *array = (void *)image; - // Invalidate all regions that depend on the cme or cc - for (int32_t i = 0; i < array->size; i++) { - struct compiled_region *region = &array->data[i]; - const struct rb_iseq_constant_body *body = region->iseq->body; - RUBY_ASSERT((unsigned int)region->start_idx < body->iseq_size); - // Restore region address to interpreter address in bytecode sequence - if (body->iseq_encoded[region->start_idx] == (VALUE)region->code) { - const void *const *code_threading_table = rb_vm_get_insns_address_table(); - int opcode = rb_vm_insn_addr2insn(region->code); - body->iseq_encoded[region->start_idx] = (VALUE)code_threading_table[opcode]; - if (UJIT_DUMP_MODE > 0) { - fprintf(stderr, "cc_or_cme=%p now out of date. Restored idx=%u in iseq=%p\n", (void *)cme_or_cc, (unsigned)region->start_idx, (void *)region->iseq); - } - } - } - - array->size = 0; - } -} - -void -rb_ujit_init(void) +ujit_init_codegen(void) { - if (!ujit_scrape_successful || !PLATFORM_SUPPORTED_P) - { - return; - } - - rb_ujit_enabled = true; // Initialize the code blocks size_t mem_size = 128 * 1024 * 1024; uint8_t* mem_block = alloc_exec_mem(mem_size); @@ -1230,9 +892,4 @@ rb_ujit_init(void) st_insert(gen_fns, (st_data_t)BIN(opt_minus), (st_data_t)&gen_opt_minus); st_insert(gen_fns, (st_data_t)BIN(opt_plus), (st_data_t)&gen_opt_plus); st_insert(gen_fns, (st_data_t)BIN(opt_send_without_block), (st_data_t)&gen_opt_send_without_block); - - method_lookup_dependency = st_init_numtable(); - struct ujit_root_struct *root; - VALUE ujit_root = TypedData_Make_Struct(0, struct ujit_root_struct, &ujit_root_type, root); - rb_gc_register_mark_object(ujit_root); } diff --git a/ujit_codegen.h b/ujit_codegen.h new file mode 100644 index 0000000000..a6667b01df --- /dev/null +++ b/ujit_codegen.h @@ -0,0 +1,10 @@ +#ifndef UJIT_CODEGEN_H +#define UJIT_CODEGEN_H 1 + +#include "stddef.h" + +uint8_t * ujit_compile_insn(const rb_iseq_t *iseq, unsigned int insn_idx, unsigned int *next_ujit_idx); + +void ujit_init_codegen(void); + +#endif // #ifndef UJIT_CODEGEN_H diff --git a/ujit_core.c b/ujit_core.c new file mode 100644 index 0000000000..5ac6407219 --- /dev/null +++ b/ujit_core.c @@ -0,0 +1,68 @@ +#include "ujit_asm.h" +#include "ujit_iface.h" +#include "ujit_core.h" + +// Get the current instruction opcode from the context object +int +ctx_get_opcode(ctx_t *ctx) +{ + return opcode_at_pc(ctx->iseq, ctx->pc); +} + +// Get an instruction argument from the context object +VALUE +ctx_get_arg(ctx_t* ctx, size_t arg_idx) +{ + assert (arg_idx + 1 < insn_len(ctx_get_opcode(ctx))); + return *(ctx->pc + arg_idx + 1); +} + +/* +Get an operand for the adjusted stack pointer address +*/ +x86opnd_t +ctx_sp_opnd(ctx_t* ctx, int32_t offset_bytes) +{ + int32_t offset = (ctx->stack_diff) * 8 + offset_bytes; + return mem_opnd(64, REG_SP, offset); +} + +/* +Make space on the stack for N values +Return a pointer to the new stack top +*/ +x86opnd_t +ctx_stack_push(ctx_t* ctx, size_t n) +{ + ctx->stack_diff += n; + + // SP points just above the topmost value + int32_t offset = (ctx->stack_diff - 1) * 8; + return mem_opnd(64, REG_SP, offset); +} + +/* +Pop N values off the stack +Return a pointer to the stack top before the pop operation +*/ +x86opnd_t +ctx_stack_pop(ctx_t* ctx, size_t n) +{ + // SP points just above the topmost value + int32_t offset = (ctx->stack_diff - 1) * 8; + x86opnd_t top = mem_opnd(64, REG_SP, offset); + + ctx->stack_diff -= n; + + return top; +} + +x86opnd_t +ctx_stack_opnd(ctx_t* ctx, int32_t idx) +{ + // SP points just above the topmost value + int32_t offset = (ctx->stack_diff - 1 - idx) * 8; + x86opnd_t opnd = mem_opnd(64, REG_SP, offset); + + return opnd; +} diff --git a/ujit_core.h b/ujit_core.h new file mode 100644 index 0000000000..ede28e7834 --- /dev/null +++ b/ujit_core.h @@ -0,0 +1,50 @@ +#ifndef UJIT_CORE_H +#define UJIT_CORE_H 1 + +#include "stddef.h" +#include "ujit_asm.h" + +// Register uJIT receives the CFP and EC into +#define REG_CFP RDI +#define REG_EC RSI + +// Register uJIT loads the SP into +#define REG_SP RDX + +// Scratch registers used by uJIT +#define REG0 RAX +#define REG1 RCX +#define REG0_32 EAX +#define REG1_32 ECX + +// Code generation context +typedef struct ctx_struct +{ + // Current PC + VALUE *pc; + + // Difference between the current stack pointer and actual stack top + int32_t stack_diff; + + // The iseq that owns the region that is compiling + const rb_iseq_t *iseq; + + // Index in the iseq of the opcode we are replacing + size_t start_idx; + + // The start of the generated code + uint8_t *code_ptr; + + // Whether we know self is a heap object + bool self_is_object; + +} ctx_t; + +int ctx_get_opcode(ctx_t *ctx); +VALUE ctx_get_arg(ctx_t* ctx, size_t arg_idx); +x86opnd_t ctx_sp_opnd(ctx_t* ctx, int32_t offset_bytes); +x86opnd_t ctx_stack_push(ctx_t* ctx, size_t n); +x86opnd_t ctx_stack_pop(ctx_t* ctx, size_t n); +x86opnd_t ctx_stack_opnd(ctx_t* ctx, int32_t idx); + +#endif // #ifndef UJIT_CORE_H diff --git a/ujit_iface.c b/ujit_iface.c new file mode 100644 index 0000000000..9b407c82d1 --- /dev/null +++ b/ujit_iface.c @@ -0,0 +1,260 @@ +#include <assert.h> +#include "insns.inc" +#include "internal.h" +#include "vm_core.h" +#include "vm_sync.h" +#include "vm_callinfo.h" +#include "builtin.h" +#include "internal/compile.h" +#include "internal/class.h" +#include "insns_info.inc" +#include "ujit.h" +#include "ujit_iface.h" +#include "ujit_codegen.h" +#include "ujit_core.h" +#include "ujit_hooks.inc" + +bool rb_ujit_enabled; + +// Hash table of encoded instructions +extern st_table *rb_encoded_insn_data; + +// Keep track of mapping from instructions to generated code +// See comment for rb_encoded_insn_data in iseq.c +void +map_addr2insn(void *code_ptr, int insn) +{ + const void * const *table = rb_vm_get_insns_address_table(); + const void * const translated_address = table[insn]; + st_data_t encoded_insn_data; + if (st_lookup(rb_encoded_insn_data, (st_data_t)translated_address, &encoded_insn_data)) { + st_insert(rb_encoded_insn_data, (st_data_t)code_ptr, encoded_insn_data); + } + else { + rb_bug("ujit: failed to find info for original instruction while dealing with addr2insn"); + } +} + +int +opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) +{ + const VALUE at_pc = *pc; + if (FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED)) { + return rb_vm_insn_addr2opcode((const void *)at_pc); + } + else { + return (int)at_pc; + } +} + +// GC root for interacting with the GC +struct ujit_root_struct {}; + +// Map cme_or_cc => [[iseq, offset]]. An entry in the map means compiled code at iseq[offset] +// is only valid when cme_or_cc is valid +static st_table *method_lookup_dependency; + +struct compiled_region_array { + int32_t size; + int32_t capa; + struct compiled_region { + const rb_iseq_t *iseq; + size_t start_idx; + uint8_t *code; + } data[]; +}; + +// Add an element to a region array, or allocate a new region array. +static struct compiled_region_array * +add_compiled_region(struct compiled_region_array *array, const rb_iseq_t *iseq, size_t start_idx, uint8_t *code) +{ + if (!array) { + // Allocate a brand new array with space for one + array = malloc(sizeof(*array) + sizeof(struct compiled_region)); + if (!array) { + return NULL; + } + array->size = 0; + array->capa = 1; + } + if (array->size == INT32_MAX) { + return NULL; + } + // Check if the region is already present + for (int32_t i = 0; i < array->size; i++) { + if (array->data[i].iseq == iseq && array->data[i].start_idx == start_idx) { + return array; + } + } + if (array->size + 1 > array->capa) { + // Double the array's capacity. + int64_t double_capa = ((int64_t)array->capa) * 2; + int32_t new_capa = (int32_t)double_capa; + if (new_capa != double_capa) { + return NULL; + } + array = realloc(array, sizeof(*array) + new_capa * sizeof(struct compiled_region)); + if (array == NULL) { + return NULL; + } + array->capa = new_capa; + } + + int32_t size = array->size; + array->data[size].iseq = iseq; + array->data[size].start_idx = start_idx; + array->data[size].code = code; + array->size++; + return array; +} + +static int +add_lookup_dependency_i(st_data_t *key, st_data_t *value, st_data_t data, int existing) +{ + ctx_t *ctx = (ctx_t *)data; + struct compiled_region_array *regions = NULL; + if (existing) { + regions = (struct compiled_region_array *)*value; + } + regions = add_compiled_region(regions, ctx->iseq, ctx->start_idx, ctx->code_ptr); + if (!regions) { + rb_bug("ujit: failed to add method lookup dependency"); // TODO: we could bail out of compiling instead + } + *value = (st_data_t)regions; + return ST_CONTINUE; +} + +// Remember that the currently compiling region is only valid while cme and cc are valid +void +assume_method_lookup_stable(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme, ctx_t *ctx) +{ + st_update(method_lookup_dependency, (st_data_t)cme, add_lookup_dependency_i, (st_data_t)ctx); + st_update(method_lookup_dependency, (st_data_t)cc, add_lookup_dependency_i, (st_data_t)ctx); + // FIXME: This is a leak! When either the cme or the cc become invalid, the other also needs to go +} + +static int +ujit_root_mark_i(st_data_t k, st_data_t v, st_data_t ignore) +{ + // FIXME: This leaks everything that end up in the dependency table! + // One way to deal with this is with weak references... + rb_gc_mark((VALUE)k); + struct compiled_region_array *regions = (void *)v; + for (int32_t i = 0; i < regions->size; i++) { + rb_gc_mark((VALUE)regions->data[i].iseq); + } + + return ST_CONTINUE; +} + +// GC callback during mark phase +static void +ujit_root_mark(void *ptr) +{ + if (method_lookup_dependency) { + st_foreach(method_lookup_dependency, ujit_root_mark_i, 0); + } +} + +static void +ujit_root_free(void *ptr) +{ + // Do nothing. The root lives as long as the process. +} + +static size_t +ujit_root_memsize(const void *ptr) +{ + // Count off-gc-heap allocation size of the dependency table + return st_memsize(method_lookup_dependency); // TODO: more accurate accounting +} + +// Custom type for interacting with the GC +// TODO: compaction support +// TODO: make this write barrier protected +static const rb_data_type_t ujit_root_type = { + "ujit_root", + {ujit_root_mark, ujit_root_free, ujit_root_memsize, }, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY +}; + +// Callback when cme or cc become invalid +void +rb_ujit_method_lookup_change(VALUE cme_or_cc) +{ + if (!method_lookup_dependency) return; + + RUBY_ASSERT(IMEMO_TYPE_P(cme_or_cc, imemo_ment) || IMEMO_TYPE_P(cme_or_cc, imemo_callcache)); + + st_data_t image; + if (st_lookup(method_lookup_dependency, (st_data_t)cme_or_cc, &image)) { + struct compiled_region_array *array = (void *)image; + // Invalidate all regions that depend on the cme or cc + for (int32_t i = 0; i < array->size; i++) { + struct compiled_region *region = &array->data[i]; + const struct rb_iseq_constant_body *body = region->iseq->body; + RUBY_ASSERT((unsigned int)region->start_idx < body->iseq_size); + // Restore region address to interpreter address in bytecode sequence + if (body->iseq_encoded[region->start_idx] == (VALUE)region->code) { + const void *const *code_threading_table = rb_vm_get_insns_address_table(); + int opcode = rb_vm_insn_addr2insn(region->code); + body->iseq_encoded[region->start_idx] = (VALUE)code_threading_table[opcode]; + if (UJIT_DUMP_MODE > 0) { + fprintf(stderr, "cc_or_cme=%p now out of date. Restored idx=%u in iseq=%p\n", (void *)cme_or_cc, (unsigned)region->start_idx, (void *)region->iseq); + } + } + } + + array->size = 0; + } +} + +void +rb_ujit_compile_iseq(const rb_iseq_t *iseq) +{ +#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE + RB_VM_LOCK_ENTER(); + VALUE *encoded = (VALUE *)iseq->body->iseq_encoded; + + unsigned int insn_idx; + unsigned int next_ujit_idx = 0; + + for (insn_idx = 0; insn_idx < iseq->body->iseq_size; /* */) { + int insn = opcode_at_pc(iseq, &encoded[insn_idx]); + int len = insn_len(insn); + + uint8_t *native_code_ptr = NULL; + + // If ujit hasn't already compiled this instruction + if (insn_idx >= next_ujit_idx) { + native_code_ptr = ujit_compile_insn(iseq, insn_idx, &next_ujit_idx); + } + + if (native_code_ptr) { + encoded[insn_idx] = (VALUE)native_code_ptr; + } + insn_idx += len; + } + RB_VM_LOCK_LEAVE(); +#endif +} + +void +rb_ujit_init(void) +{ + if (!ujit_scrape_successful || !PLATFORM_SUPPORTED_P) + { + return; + } + + rb_ujit_enabled = true; + + // Initialize ujit codegen + ujit_init_codegen(); + + // Initialize the GC hooks + method_lookup_dependency = st_init_numtable(); + struct ujit_root_struct *root; + VALUE ujit_root = TypedData_Make_Struct(0, struct ujit_root_struct, &ujit_root_type, root); + rb_gc_register_mark_object(ujit_root); +} diff --git a/ujit_iface.h b/ujit_iface.h new file mode 100644 index 0000000000..1e43c7c60c --- /dev/null +++ b/ujit_iface.h @@ -0,0 +1,27 @@ +// +// These are definitions uJIT uses to interface with the CRuby codebase, +// but which are only used internally by uJIT. +// + +#ifndef UJIT_IFACE_H +#define UJIT_IFACE_H 1 + +#include "stddef.h" +#include "stdint.h" +#include "stdbool.h" +#include "internal.h" +#include "vm_core.h" +#include "vm_callinfo.h" +#include "builtin.h" +#include "ujit_core.h" + +#ifndef rb_callcache +struct rb_callcache; +#define rb_callcache rb_callcache +#endif + +void map_addr2insn(void *code_ptr, int insn); +int opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc); +void assume_method_lookup_stable(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme, ctx_t *ctx); + +#endif // #ifndef UJIT_IFACE_H @@ -13,7 +13,7 @@ #include "version.h" #include "vm_core.h" #include "mjit.h" -#include "ujit_compile.h" +#include "ujit_iface.h" #include <stdio.h> #ifndef EXIT_SUCCESS @@ -126,7 +126,7 @@ ruby_show_version(void) } if (rb_ujit_enabled_p()) { - fputs("MicroJIT is on\n", stdout); + fputs("uJIT is enabled\n", stdout); } #ifdef RUBY_LAST_COMMIT_TITLE fputs("last_commit=" RUBY_LAST_COMMIT_TITLE, stdout); diff --git a/vm_method.c b/vm_method.c index 55fb2b5337..544abca753 100644 --- a/vm_method.c +++ b/vm_method.c @@ -3,7 +3,7 @@ */ #include "id_table.h" -#include "ujit_compile.h" +#include "ujit.h" #define METHOD_DEBUG 0 diff --git a/win32/Makefile.sub b/win32/Makefile.sub index 0dd62993e6..241489d474 100644 --- a/win32/Makefile.sub +++ b/win32/Makefile.sub @@ -1347,7 +1347,7 @@ $(MJIT_PRECOMPILED_HEADER): $(MJIT_PRECOMPILED_HEADER_NAME) $(Q) $(MAKE_LINK) $(MJIT_PRECOMPILED_HEADER_NAME:.pch=.pdb) $(arch_hdrdir)/$(MJIT_PRECOMPILED_HEADER_NAME:.pch=.pdb) INSNS = opt_sc.inc optinsn.inc optunifs.inc insns.inc insns_info.inc \ - vmtc.inc vm.inc mjit_compile.inc ujit_examples.inc + vmtc.inc vm.inc mjit_compile.inc ujit_hooks.inc !if [exit > insns_rules.mk] !else if [for %I in ($(INSNS)) do \ |