diff options
| -rw-r--r-- | insns.def | 2 | ||||
| -rw-r--r-- | mjit.c | 26 | ||||
| -rw-r--r-- | mjit.h | 7 | ||||
| -rw-r--r-- | mjit_compile.c | 4 | ||||
| -rw-r--r-- | tool/ruby_vm/views/_mjit_compile_getinlinecache.erb | 36 | ||||
| -rw-r--r-- | tool/ruby_vm/views/mjit_compile.inc.erb | 2 | ||||
| -rw-r--r-- | vm_insnhelper.c | 11 |
7 files changed, 82 insertions, 6 deletions
@@ -1023,7 +1023,7 @@ opt_getinlinecache () (VALUE val) { - if (vm_ic_hit_p(ic, GET_EP())) { + if (vm_ic_hit_p(ic->ic_serial, ic->ic_cref, GET_EP())) { val = ic->value; JUMP(dst); } @@ -82,6 +82,24 @@ mjit_gc_exit_hook(void) CRITICAL_SECTION_FINISH(4, "mjit_gc_exit_hook"); } +// Lock setinlinecache +void +rb_mjit_before_vm_ic_update(void) +{ + if (!mjit_enabled) + return; + CRITICAL_SECTION_START(3, "before vm_ic_update"); +} + +// Unlock setinlinecache +void +rb_mjit_after_vm_ic_update(void) +{ + if (!mjit_enabled) + return; + CRITICAL_SECTION_FINISH(3, "after vm_ic_update"); +} + // Deal with ISeq movement from compactor void mjit_update_references(const rb_iseq_t *iseq) @@ -378,6 +396,14 @@ rb_mjit_recompile_inlining(const rb_iseq_t *iseq) mjit_recompile(iseq); } +// Recompile iseq, disabling getconstant inlining +void +rb_mjit_recompile_const(const rb_iseq_t *iseq) +{ + rb_mjit_iseq_compile_info(iseq->body)->disable_const_cache = true; + mjit_recompile(iseq); +} + extern VALUE ruby_archlibdir_path, ruby_prefix_path; // Initialize header_file, pch_file, libruby_pathflag. Return true on success. @@ -70,6 +70,8 @@ struct rb_mjit_compile_info { bool disable_send_cache; // Disable method inlining bool disable_inlining; + // Disable opt_getinlinecache inlining + bool disable_const_cache; }; typedef VALUE (*mjit_func_t)(rb_execution_context_t *, rb_control_frame_t *); @@ -81,10 +83,13 @@ RUBY_EXTERN bool mjit_call_p; extern void rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq); extern VALUE rb_mjit_wait_call(rb_execution_context_t *ec, struct rb_iseq_constant_body *body); extern struct rb_mjit_compile_info* rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body); +extern void rb_mjit_before_vm_ic_update(void); +extern void rb_mjit_after_vm_ic_update(void); extern void rb_mjit_recompile_send(const rb_iseq_t *iseq); extern void rb_mjit_recompile_ivar(const rb_iseq_t *iseq); extern void rb_mjit_recompile_exivar(const rb_iseq_t *iseq); extern void rb_mjit_recompile_inlining(const rb_iseq_t *iseq); +extern void rb_mjit_recompile_const(const rb_iseq_t *iseq); RUBY_SYMBOL_EXPORT_END extern bool mjit_compile(FILE *f, const rb_iseq_t *iseq, const char *funcname, int id); @@ -189,6 +194,8 @@ void mjit_finish(bool close_handle_p); # else // USE_MJIT +static inline void rb_mjit_before_vm_ic_update(void){} +static inline void rb_mjit_after_vm_ic_update(void){} static inline struct mjit_cont *mjit_cont_new(rb_execution_context_t *ec){return NULL;} static inline void mjit_cont_free(struct mjit_cont *cont){} static inline void mjit_gc_start_hook(void){} diff --git a/mjit_compile.c b/mjit_compile.c index 8591b5c5d6..7e715b7e10 100644 --- a/mjit_compile.c +++ b/mjit_compile.c @@ -307,6 +307,10 @@ compile_cancel_handler(FILE *f, const struct rb_iseq_constant_body *body, struct fprintf(f, " rb_mjit_recompile_exivar(original_iseq);\n"); fprintf(f, " goto cancel;\n"); + fprintf(f, "\nconst_cancel:\n"); + fprintf(f, " rb_mjit_recompile_const(original_iseq);\n"); + fprintf(f, " goto cancel;\n"); + fprintf(f, "\ncancel:\n"); fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel);\n"); if (status->local_stack_p) { diff --git a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb b/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb new file mode 100644 index 0000000000..44b7f3286a --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb @@ -0,0 +1,36 @@ +% # -*- C -*- +% # Copyright (c) 2020 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% # compiler: Declare dst and ic +% insn.opes.each_with_index do |ope, i| + <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; +% end + +% # compiler: Capture IC values, locking getinlinecache + rb_mjit_before_vm_ic_update(); + rb_serial_t ic_serial = ic->ic_serial; + const rb_cref_t *ic_cref = ic->ic_cref; + VALUE ic_value = ic->value; + rb_mjit_after_vm_ic_update(); + + if (ic_serial && !status->compile_info->disable_const_cache) { +% # JIT: Inline everything in IC, and cancel the slow path + fprintf(f, " if (vm_ic_hit_p((rb_serial_t)%"PRI_SERIALT_PREFIX"u, (const rb_cref_t *)0x%"PRIxVALUE", reg_cfp->ep)) {", ic_serial, (VALUE)ic_cref); + fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ic_value); + fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst); + fprintf(f, " }"); + fprintf(f, " else {"); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " goto const_cancel;\n"); + fprintf(f, " }"); + +% # compiler: Move JIT compiler's internal stack pointer + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + break; + } diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb index a7068c8532..f622467538 100644 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ b/tool/ruby_vm/views/mjit_compile.inc.erb @@ -65,6 +65,8 @@ switch (insn) { <%= render 'mjit_compile_ivar', locals: { insn: insn } -%> % when 'invokebuiltin', 'opt_invokebuiltin_delegate' <%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> +% when 'opt_getinlinecache' +<%= render 'mjit_compile_getinlinecache', locals: { insn: insn } -%> % when 'leave', 'opt_invokebuiltin_delegate_leave' % # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining. % if insn.name == 'opt_invokebuiltin_delegate_leave' diff --git a/vm_insnhelper.c b/vm_insnhelper.c index e8c9719155..57f85b9a31 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -4577,12 +4577,11 @@ vm_opt_newarray_min(rb_num_t num, const VALUE *ptr) #undef id_cmp static int -vm_ic_hit_p(IC ic, const VALUE *reg_ep) +vm_ic_hit_p(const rb_serial_t ic_serial, const rb_cref_t *ic_cref, const VALUE *reg_ep) { - if (ic->ic_serial == GET_GLOBAL_CONSTANT_STATE() && - rb_ractor_main_p()) { - return (ic->ic_cref == NULL || // no need to check CREF - ic->ic_cref == vm_get_cref(reg_ep)); + if (ic_serial == GET_GLOBAL_CONSTANT_STATE() && rb_ractor_main_p()) { + return (ic_cref == NULL || // no need to check CREF + ic_cref == vm_get_cref(reg_ep)); } return FALSE; } @@ -4591,9 +4590,11 @@ static void vm_ic_update(IC ic, VALUE val, const VALUE *reg_ep) { VM_ASSERT(ic->value != Qundef); + rb_mjit_before_vm_ic_update(); ic->value = val; ic->ic_serial = GET_GLOBAL_CONSTANT_STATE() - ruby_vm_const_missing_count; ic->ic_cref = vm_get_const_key_cref(reg_ep); + rb_mjit_after_vm_ic_update(); ruby_vm_const_missing_count = 0; } |