summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--benchmark/vm_const.yml6
-rw-r--r--common.mk2
-rw-r--r--compile.c205
-rw-r--r--insns.def63
-rw-r--r--iseq.c140
-rw-r--r--iseq.h1
-rw-r--r--test/ruby/test_mjit.rb6
-rw-r--r--test/ruby/test_yjit.rb8
-rw-r--r--tool/ruby_vm/views/_mjit_compile_getconstant_path.erb (renamed from tool/ruby_vm/views/_mjit_compile_getinlinecache.erb)1
-rw-r--r--tool/ruby_vm/views/mjit_compile.inc.erb4
-rw-r--r--vm_core.h19
-rw-r--r--vm_insnhelper.c82
-rw-r--r--yjit.h4
-rw-r--r--yjit/src/codegen.rs29
-rw-r--r--yjit/src/cruby_bindings.inc.rs280
-rw-r--r--yjit/src/invariants.rs82
16 files changed, 482 insertions, 450 deletions
diff --git a/benchmark/vm_const.yml b/benchmark/vm_const.yml
index 6064d4eed0..8939ca0cd3 100644
--- a/benchmark/vm_const.yml
+++ b/benchmark/vm_const.yml
@@ -1,7 +1,13 @@
prelude: |
Const = 1
+ A = B = C = D = E = F = G = H = I = J = K = L = M = N = O = P = Q = R = S = T = U = V = W = X = Y = Z = 1
+ def foo
+ A; B; C; D; E; F; G; H; I; J; K; L; M; N; O; P; Q; R; S; T; U; V; W; X; Y; Z
+ end
benchmark:
vm_const: |
j = Const
k = Const
+ vm_const_many: |
+ foo
loop_count: 30000000
diff --git a/common.mk b/common.mk
index 3d3976d5d5..86226b58ca 100644
--- a/common.mk
+++ b/common.mk
@@ -1045,7 +1045,7 @@ $(srcs_vpath)mjit_compile.inc: $(tooldir)/ruby_vm/views/mjit_compile.inc.erb $(i
$(tooldir)/ruby_vm/views/_mjit_compile_insn.erb $(tooldir)/ruby_vm/views/_mjit_compile_send.erb \
$(tooldir)/ruby_vm/views/_mjit_compile_ivar.erb \
$(tooldir)/ruby_vm/views/_mjit_compile_insn_body.erb $(tooldir)/ruby_vm/views/_mjit_compile_pc_and_sp.erb \
- $(tooldir)/ruby_vm/views/_mjit_compile_invokebuiltin.erb $(tooldir)/ruby_vm/views/_mjit_compile_getinlinecache.erb
+ $(tooldir)/ruby_vm/views/_mjit_compile_invokebuiltin.erb $(tooldir)/ruby_vm/views/_mjit_compile_getconstant_path.erb
BUILTIN_RB_SRCS = \
$(srcdir)/ast.rb \
diff --git a/compile.c b/compile.c
index e906bd1e10..3e51c5cd53 100644
--- a/compile.c
+++ b/compile.c
@@ -2251,6 +2251,30 @@ add_adjust_info(struct iseq_insn_info_entry *insns_info, unsigned int *positions
return TRUE;
}
+static ID *
+array_to_idlist(VALUE arr)
+{
+ RUBY_ASSERT(RB_TYPE_P(arr, T_ARRAY));
+ long size = RARRAY_LEN(arr);
+ ID *ids = (ID *)ALLOC_N(ID, size + 1);
+ for (int i = 0; i < size; i++) {
+ VALUE sym = RARRAY_AREF(arr, i);
+ ids[i] = SYM2ID(sym);
+ }
+ ids[size] = 0;
+ return ids;
+}
+
+static VALUE
+idlist_to_array(const ID *ids)
+{
+ VALUE arr = rb_ary_new();
+ while (*ids) {
+ rb_ary_push(arr, ID2SYM(*ids++));
+ }
+ return arr;
+}
+
/**
ruby insn object list -> raw instruction sequence
*/
@@ -2433,6 +2457,21 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
}
/* [ TS_IVC | TS_ICVARC | TS_ISE | TS_IC ] */
case TS_IC: /* inline cache: constants */
+ {
+ unsigned int ic_index = ISEQ_COMPILE_DATA(iseq)->ic_index++;
+ IC ic = &ISEQ_IS_ENTRY_START(body, type)[ic_index].ic_cache;
+ if (UNLIKELY(ic_index >= body->ic_size)) {
+ BADINSN_DUMP(anchor, &iobj->link, 0);
+ COMPILE_ERROR(iseq, iobj->insn_info.line_no,
+ "iseq_set_sequence: ic_index overflow: index: %d, size: %d",
+ ic_index, ISEQ_IS_SIZE(body));
+ }
+
+ ic->segments = array_to_idlist(operands[j]);
+
+ generated_iseq[code_index + 1 + j] = (VALUE)ic;
+ }
+ break;
case TS_ISE: /* inline storage entry: `once` insn */
case TS_ICVARC: /* inline cvar cache */
case TS_IVC: /* inline ivar cache */
@@ -2447,11 +2486,6 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
}
generated_iseq[code_index + 1 + j] = (VALUE)ic;
- if (insn == BIN(opt_getinlinecache) && type == TS_IC) {
- // Store the instruction index for opt_getinlinecache on the IC for
- // YJIT to invalidate code when opt_setinlinecache runs.
- ic->get_insn_idx = (unsigned int)code_index;
- }
break;
}
case TS_CALLDATA:
@@ -5233,6 +5267,30 @@ compile_massign(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node,
return COMPILE_OK;
}
+static VALUE
+collect_const_segments(rb_iseq_t *iseq, const NODE *node)
+{
+ VALUE arr = rb_ary_new();
+ for (;;)
+ {
+ switch (nd_type(node)) {
+ case NODE_CONST:
+ rb_ary_unshift(arr, ID2SYM(node->nd_vid));
+ return arr;
+ case NODE_COLON3:
+ rb_ary_unshift(arr, ID2SYM(node->nd_mid));
+ rb_ary_unshift(arr, ID2SYM(idNULL));
+ return arr;
+ case NODE_COLON2:
+ rb_ary_unshift(arr, ID2SYM(node->nd_mid));
+ node = node->nd_head;
+ break;
+ default:
+ return Qfalse;
+ }
+ }
+}
+
static int
compile_const_prefix(rb_iseq_t *iseq, const NODE *const node,
LINK_ANCHOR *const pref, LINK_ANCHOR *const body)
@@ -8970,37 +9028,31 @@ compile_match(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i
static int
compile_colon2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped)
{
- const int line = nd_line(node);
if (rb_is_const_id(node->nd_mid)) {
/* constant */
- LABEL *lend = NEW_LABEL(line);
- int ic_index = ISEQ_BODY(iseq)->ic_size++;
-
- DECL_ANCHOR(pref);
- DECL_ANCHOR(body);
-
- INIT_ANCHOR(pref);
- INIT_ANCHOR(body);
- CHECK(compile_const_prefix(iseq, node, pref, body));
- if (LIST_INSN_SIZE_ZERO(pref)) {
- if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) {
- ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index));
- }
- else {
+ VALUE segments;
+ if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache &&
+ (segments = collect_const_segments(iseq, node))) {
+ ISEQ_BODY(iseq)->ic_size++;
+ ADD_INSN1(ret, node, opt_getconstant_path, segments);
+ RB_OBJ_WRITTEN(iseq, Qundef, segments);
+ } else {
+ /* constant */
+ DECL_ANCHOR(pref);
+ DECL_ANCHOR(body);
+
+ INIT_ANCHOR(pref);
+ INIT_ANCHOR(body);
+ CHECK(compile_const_prefix(iseq, node, pref, body));
+ if (LIST_INSN_SIZE_ZERO(pref)) {
ADD_INSN(ret, node, putnil);
+ ADD_SEQ(ret, body);
}
-
- ADD_SEQ(ret, body);
-
- if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) {
- ADD_INSN1(ret, node, opt_setinlinecache, INT2FIX(ic_index));
- ADD_LABEL(ret, lend);
+ else {
+ ADD_SEQ(ret, pref);
+ ADD_SEQ(ret, body);
}
}
- else {
- ADD_SEQ(ret, pref);
- ADD_SEQ(ret, body);
- }
}
else {
/* function call */
@@ -9017,25 +9069,18 @@ compile_colon2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node,
static int
compile_colon3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped)
{
- const int line = nd_line(node);
- LABEL *lend = NEW_LABEL(line);
- int ic_index = ISEQ_BODY(iseq)->ic_size++;
-
debugi("colon3#nd_mid", node->nd_mid);
/* add cache insn */
if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) {
- ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index));
- ADD_INSN(ret, node, pop);
- }
-
- ADD_INSN1(ret, node, putobject, rb_cObject);
- ADD_INSN1(ret, node, putobject, Qtrue);
- ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_mid));
-
- if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) {
- ADD_INSN1(ret, node, opt_setinlinecache, INT2FIX(ic_index));
- ADD_LABEL(ret, lend);
+ ISEQ_BODY(iseq)->ic_size++;
+ VALUE segments = rb_ary_new_from_args(2, ID2SYM(idNULL), ID2SYM(node->nd_mid));
+ ADD_INSN1(ret, node, opt_getconstant_path, segments);
+ RB_OBJ_WRITTEN(iseq, Qundef, segments);
+ } else {
+ ADD_INSN1(ret, node, putobject, rb_cObject);
+ ADD_INSN1(ret, node, putobject, Qtrue);
+ ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_mid));
}
if (popped) {
@@ -9536,18 +9581,14 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no
case NODE_CONST:{
debugi("nd_vid", node->nd_vid);
- if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) {
- LABEL *lend = NEW_LABEL(line);
- int ic_index = body->ic_size++;
-
- ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index));
- ADD_INSN1(ret, node, putobject, Qtrue);
- ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_vid));
- ADD_INSN1(ret, node, opt_setinlinecache, INT2FIX(ic_index));
- ADD_LABEL(ret, lend);
- }
- else {
- ADD_INSN(ret, node, putnil);
+ if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) {
+ body->ic_size++;
+ VALUE segments = rb_ary_new_from_args(1, ID2SYM(node->nd_vid));
+ ADD_INSN1(ret, node, opt_getconstant_path, segments);
+ RB_OBJ_WRITTEN(iseq, Qundef, segments);
+ }
+ else {
+ ADD_INSN(ret, node, putnil);
ADD_INSN1(ret, node, putobject, Qtrue);
ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_vid));
}
@@ -10032,10 +10073,16 @@ insn_data_to_s_detail(INSN *iobj)
rb_str_concat(str, opobj_inspect(OPERAND_AT(iobj, j)));
break;
case TS_IC: /* inline cache */
+ rb_str_concat(str, opobj_inspect(OPERAND_AT(iobj, j)));
+ break;
case TS_IVC: /* inline ivar cache */
+ rb_str_catf(str, "<ivc:%d>", FIX2INT(OPERAND_AT(iobj, j)));
+ break;
case TS_ICVARC: /* inline cvar cache */
+ rb_str_catf(str, "<icvarc:%d>", FIX2INT(OPERAND_AT(iobj, j)));
+ break;
case TS_ISE: /* inline storage entry */
- rb_str_catf(str, "<ic:%d>", FIX2INT(OPERAND_AT(iobj, j)));
+ rb_str_catf(str, "<ise:%d>", FIX2INT(OPERAND_AT(iobj, j)));
break;
case TS_CALLDATA: /* we store these as call infos at compile time */
{
@@ -10431,9 +10478,20 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor,
}
break;
case TS_IC:
- argv[j] = op;
- if (NUM2UINT(op) >= ISEQ_BODY(iseq)->ic_size) {
- ISEQ_BODY(iseq)->ic_size = NUM2INT(op) + 1;
+ {
+ VALUE segments = rb_ary_new();
+ op = rb_to_array_type(op);
+
+ for (int i = 0; i < RARRAY_LEN(op); i++) {
+ VALUE sym = RARRAY_AREF(op, i);
+ sym = rb_to_symbol_type(sym);
+ rb_ary_push(segments, sym);
+ }
+
+ RB_GC_GUARD(op);
+ argv[j] = segments;
+ RB_OBJ_WRITTEN(iseq, Qundef, segments);
+ ISEQ_BODY(iseq)->ic_size++;
}
break;
case TS_IVC: /* inline ivar cache */
@@ -10627,6 +10685,7 @@ rb_iseq_mark_insn_storage(struct iseq_compile_data_storage *storage)
case TS_CDHASH:
case TS_ISEQ:
case TS_VALUE:
+ case TS_IC: // constant path array
case TS_CALLDATA: // ci is stored.
{
VALUE op = OPERAND_AT(iobj, j);
@@ -11255,6 +11314,12 @@ ibf_dump_code(struct ibf_dump *dump, const rb_iseq_t *iseq)
wv = (VALUE)ibf_dump_iseq(dump, (const rb_iseq_t *)op);
break;
case TS_IC:
+ {
+ IC ic = (IC)op;
+ VALUE arr = idlist_to_array(ic->segments);
+ wv = ibf_dump_object(dump, arr);
+ }
+ break;
case TS_ISE:
case TS_IVC:
case TS_ICVARC:
@@ -11299,6 +11364,7 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod
struct rb_iseq_constant_body *load_body = ISEQ_BODY(iseq);
struct rb_call_data *cd_entries = load_body->call_data;
+ int ic_index = 0;
iseq_bits_t * mark_offset_bits;
@@ -11315,7 +11381,6 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod
for (code_index=0; code_index<iseq_size;) {
/* opcode */
const VALUE insn = code[code_index] = ibf_load_small_value(load, &reading_pos);
- const unsigned int insn_index = code_index;
const char *types = insn_op_types(insn);
int op_index;
@@ -11370,6 +11435,16 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod
break;
}
case TS_IC:
+ {
+ VALUE op = ibf_load_small_value(load, &reading_pos);
+ VALUE arr = ibf_load_object(load, op);
+
+ IC ic = &ISEQ_IS_IC_ENTRY(load_body, ic_index++);
+ ic->segments = array_to_idlist(arr);
+
+ code[code_index] = (VALUE)ic;
+ }
+ break;
case TS_ISE:
case TS_ICVARC:
case TS_IVC:
@@ -11378,12 +11453,6 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod
ISE ic = ISEQ_IS_ENTRY_START(load_body, operand_type) + op;
code[code_index] = (VALUE)ic;
-
- if (insn == BIN(opt_getinlinecache) && operand_type == TS_IC) {
- // Store the instruction index for opt_getinlinecache on the IC for
- // YJIT to invalidate code when opt_setinlinecache runs.
- ic->ic_cache.get_insn_idx = insn_index;
- }
}
break;
case TS_CALLDATA:
diff --git a/insns.def b/insns.def
index 15c4734b8b..cde327e430 100644
--- a/insns.def
+++ b/insns.def
@@ -253,6 +253,29 @@ setclassvariable
vm_setclassvariable(GET_ISEQ(), GET_CFP(), id, val, ic);
}
+DEFINE_INSN
+opt_getconstant_path
+(IC ic)
+()
+(VALUE val)
+// attr bool leaf = false; /* may autoload or raise */
+{
+ const ID *segments = ic->segments;
+ struct iseq_inline_constant_cache_entry *ice = ic->entry;
+ if (ice && vm_ic_hit_p(ice, GET_EP())) {
+ val = ice->value;
+
+ VM_ASSERT(val == vm_get_ev_const_chain(ec, segments));
+ } else {
+ ruby_vm_constant_cache_misses++;
+ val = vm_get_ev_const_chain(ec, segments);
+ vm_ic_track_const_chain(GET_CFP(), ic, segments);
+ // Because leaf=false, we need to undo the PC increment to get the address to this instruction
+ // INSN_ATTR(width) == 2
+ vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2);
+ }
+}
+
/* Get constant variable id. If klass is Qnil and allow_nil is Qtrue, constants
are searched in the current scope. Otherwise, get constant under klass
class or module.
@@ -1039,46 +1062,6 @@ branchnil
/* for optimize */
/**********************************************************/
-/* push inline-cached value and go to dst if it is valid */
-DEFINE_INSN
-opt_getinlinecache
-(OFFSET dst, IC ic)
-()
-(VALUE val)
-{
- struct iseq_inline_constant_cache_entry *ice = ic->entry;
-
- // If there isn't an entry, then we're going to walk through the ISEQ
- // starting at this instruction until we get to the associated
- // opt_setinlinecache and associate this inline cache with every getconstant
- // listed in between. We're doing this here instead of when the instructions
- // are first compiled because it's possible to turn off inline caches and we
- // want this to work in either case.
- if (!ice) {
- vm_ic_compile(GET_CFP(), ic);
- }
-
- if (ice && vm_ic_hit_p(ice, GET_EP())) {
- val = ice->value;
- JUMP(dst);
- }
- else {
- ruby_vm_constant_cache_misses++;
- val = Qnil;
- }
-}
-
-/* set inline cache */
-DEFINE_INSN
-opt_setinlinecache
-(IC ic)
-(VALUE val)
-(VALUE val)
-// attr bool leaf = false;
-{
- vm_ic_update(GET_ISEQ(), ic, val, GET_EP());
-}
-
/* run iseq only once */
DEFINE_INSN
once
diff --git a/iseq.c b/iseq.c
index 4a2c9a33ee..dca41a146d 100644
--- a/iseq.c
+++ b/iseq.c
@@ -102,68 +102,49 @@ compile_data_free(struct iseq_compile_data *compile_data)
}
}
-struct iseq_clear_ic_references_data {
- IC ic;
-};
-
-// This iterator is used to walk through the instructions and clean any
-// references to ICs that are contained within this ISEQ out of the VM's
-// constant cache table. It passes around a struct that holds the current IC
-// we're looking for, which can be NULL (if we haven't hit an opt_getinlinecache
-// instruction yet) or set to an IC (if we've hit an opt_getinlinecache and
-// haven't yet hit the associated opt_setinlinecache).
-static bool
-iseq_clear_ic_references_i(VALUE *code, VALUE insn, size_t index, void *data)
-{
- struct iseq_clear_ic_references_data *ic_data = (struct iseq_clear_ic_references_data *) data;
+static void
+remove_from_constant_cache(ID id, IC ic) {
+ rb_vm_t *vm = GET_VM();
+ VALUE lookup_result;
+ st_data_t ic_data = (st_data_t)ic;
- switch (insn) {
- case BIN(opt_getinlinecache): {
- RUBY_ASSERT_ALWAYS(ic_data->ic == NULL);
+ if (rb_id_table_lookup(vm->constant_cache, id, &lookup_result)) {
+ st_table *ics = (st_table *)lookup_result;
+ st_delete(ics, &ic_data, NULL);
- ic_data->ic = (IC) code[index + 2];
- return true;
- }
- case BIN(getconstant): {
- if (ic_data->ic != NULL) {
- ID id = (ID) code[index + 1];
- rb_vm_t *vm = GET_VM();
- VALUE lookup_result;
-
- if (rb_id_table_lookup(vm->constant_cache, id, &lookup_result)) {
- st_table *ics = (st_table *)lookup_result;
- st_data_t ic = (st_data_t)ic_data->ic;
- st_delete(ics, &ic, NULL);
-
- if (ics->num_entries == 0) {
- rb_id_table_delete(vm->constant_cache, id);
- st_free_table(ics);
- }
- }
+ if (ics->num_entries == 0) {
+ rb_id_table_delete(vm->constant_cache, id);
+ st_free_table(ics);
}
-
- return true;
- }
- case BIN(opt_setinlinecache): {
- RUBY_ASSERT_ALWAYS(ic_data->ic != NULL);
-
- ic_data->ic = NULL;
- return true;
- }
- default:
- return true;
}
}
// When an ISEQ is being freed, all of its associated ICs are going to go away
-// as well. Because of this, we need to walk through the ISEQ, find any
-// opt_getinlinecache calls, and clear out the VM's constant cache of associated
-// ICs.
+// as well. Because of this, we need to iterate over the ICs, and clear them
+// from the VM's constant cache.
static void
iseq_clear_ic_references(const rb_iseq_t *iseq)
{
- struct iseq_clear_ic_references_data data = { .ic = NULL };
- rb_iseq_each(iseq, 0, iseq_clear_ic_references_i, (void *) &data);
+ for (unsigned int ic_idx = 0; ic_idx < ISEQ_BODY(iseq)->ic_size; ic_idx++) {
+ IC ic = &ISEQ_IS_IC_ENTRY(ISEQ_BODY(iseq), ic_idx);
+
+ // Iterate over the IC's constant path's segments and clean any references to
+ // the ICs out of the VM's constant cache table.
+ const ID *segments = ic->segments;
+
+ // It's possible that segments is NULL if we overallocated an IC but
+ // optimizations removed the instruction using it
+ if (segments == NULL)
+ continue;
+
+ for (int i = 0; segments[i]; i++) {
+ ID id = segments[i];
+ if (id == idNULL) continue;
+ remove_from_constant_cache(id, ic);
+ }
+
+ ruby_xfree((void *)segments);
+ }
}
void
@@ -213,32 +194,7 @@ rb_iseq_free(const rb_iseq_t *iseq)
RUBY_FREE_LEAVE("iseq");
}
-#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
-static VALUE
-rb_vm_insn_addr2insn2(const void *addr)
-{
- return (VALUE)rb_vm_insn_addr2insn(addr);
-}
-#endif
-
-// The translator for OPT_DIRECT_THREADED_CODE and OPT_CALL_THREADED_CODE does
-// some normalization to always return the non-trace version of instructions. To
-// mirror that behavior in token-threaded environments, we normalize in this
-// translator by also returning non-trace opcodes.
-static VALUE
-rb_vm_insn_normalizing_translator(const void *addr)
-{
- VALUE opcode = (VALUE)addr;
- VALUE trace_opcode_threshold = (VM_INSTRUCTION_SIZE / 2);
-
- if (opcode >= trace_opcode_threshold) {
- return opcode - trace_opcode_threshold;
- }
- return opcode;
-}
-
typedef VALUE iseq_value_itr_t(void *ctx, VALUE obj);
-typedef VALUE rb_vm_insns_translator_t(const void *addr);
static inline void
iseq_scan_bits(unsigned int page, iseq_bits_t bits, VALUE *code, iseq_value_itr_t *func, void *data)
@@ -593,6 +549,17 @@ rb_iseq_memsize(const rb_iseq_t *iseq)
/* body->is_entries */
size += ISEQ_IS_SIZE(body) * sizeof(union iseq_inline_storage_entry);
+ /* IC entries constant segments */
+ for (unsigned int ic_idx = 0; ic_idx < body->ic_size; ic_idx++) {
+ IC ic = &ISEQ_IS_IC_ENTRY(body, ic_idx);
+ const ID *ids = ic->segments;
+ if (!ids) continue;
+ while (*ids++) {
+ size += sizeof(ID);
+ }
+ size += sizeof(ID); // null terminator
+ }
+
/* body->call_data */
size += body->ci_size * sizeof(struct rb_call_data);
// TODO: should we count imemo_callinfo?
@@ -2175,6 +2142,16 @@ rb_insn_operand_intern(const rb_iseq_t *iseq,
}
case TS_IC:
+ {
+ ret = rb_sprintf("<ic:%"PRIdPTRDIFF" ", (union iseq_inline_storage_entry *)op - ISEQ_BODY(iseq)->is_entries);
+ const ID *segments = ((IC)op)->segments;
+ rb_str_cat2(ret, rb_id2name(*segments++));
+ while (*segments) {
+ rb_str_catf(ret, "::%s", rb_id2name(*segments++));
+ }
+ rb_str_cat2(ret, ">");
+ }
+ break;
case TS_IVC:
case TS_ICVARC:
case TS_ISE:
@@ -3011,6 +2988,15 @@ iseq_data_to_ary(const rb_iseq_t *iseq)
}
break;
case TS_IC:
+ {
+ VALUE list = rb_ary_new();
+ const ID *ids = ((IC)*seq)->segments;
+ while (*ids) {
+ rb_ary_push(list, ID2SYM(*ids++));
+ }
+ rb_ary_push(ary, list);
+ }
+ break;
case TS_IVC:
case TS_ICVARC:
case TS_ISE:
diff --git a/iseq.h b/iseq.h
index e7db9b951f..fa328e9a84 100644
--- a/iseq.h
+++ b/iseq.h
@@ -119,6 +119,7 @@ struct iseq_compile_data {
int node_level;
int isolated_depth;
unsigned int ci_index;
+ unsigned int ic_index;
const rb_compile_option_t *option;
struct rb_id_table *ivar_cache_table;
const struct rb_builtin_function *builtin_function_table;
diff --git a/test/ruby/test_mjit.rb b/test/ruby/test_mjit.rb
index 9cd93855bd..e49195f763 100644
--- a/test/ruby/test_mjit.rb
+++ b/test/ruby/test_mjit.rb
@@ -179,7 +179,7 @@ class TestMJIT < Test::Unit::TestCase
end
def test_compile_insn_constant
- assert_compile_once("#{<<~"begin;"}\n#{<<~"end;"}", result_inspect: '1', insns: %i[getconstant setconstant])
+ assert_compile_once("#{<<~"begin;"}\n#{<<~"end;"}", result_inspect: '1', insns: %i[opt_getconstant_path setconstant])
begin;
FOO = 1
FOO
@@ -490,8 +490,8 @@ class TestMJIT < Test::Unit::TestCase
end;
end
- def test_compile_insn_inlinecache
- assert_compile_once('Struct', result_inspect: 'Struct', insns: %i[opt_getinlinecache opt_setinlinecache])
+ def test_compile_insn_getconstant_path
+ assert_compile_once('Struct', result_inspect: 'Struct', insns: %i[opt_getconstant_path])
end
def test_compile_insn_once
diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb
index 37e72dcafa..6cafb21698 100644
--- a/test/ruby/test_yjit.rb
+++ b/test/ruby/test_yjit.rb
@@ -389,8 +389,8 @@ class TestYJIT < Test::Unit::TestCase
assert_compiles("'foo' =~ /(o)./; $2", insns: %i[getspecial], result: nil)
end
- def test_compile_opt_getinlinecache
- assert_compiles(<<~RUBY, insns: %i[opt_getinlinecache], result: 123, call_threshold: 2)
+ def test_compile_opt_getconstant_path
+ assert_compiles(<<~RUBY, insns: %i[opt_getconstant_path], result: 123, call_threshold: 2)
def get_foo
FOO
end
@@ -402,8 +402,8 @@ class TestYJIT < Test::Unit::TestCase
RUBY
end
- def test_opt_getinlinecache_slowpath
- assert_compiles(<<~RUBY, exits: { opt_getinlinecache: 1 }, result: [42, 42, 1, 1], call_threshold: 2)
+ def test_opt_getconstant_path_slowpath
+ assert_compiles(<<~RUBY, exits: { opt_getconstant_path: 1 }, result: [42, 42, 1, 1], call_threshold: 2)
class A
FOO = 42
class << self
diff --git a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb b/tool/ruby_vm/views/_mjit_compile_getconstant_path.erb
index fa38af4045..c321da9a52 100644
--- a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb
+++ b/tool/ruby_vm/views/_mjit_compile_getconstant_path.erb
@@ -17,7 +17,6 @@
% # JIT: Inline everything in IC, and cancel the slow path
fprintf(f, " if (vm_inlined_ic_hit_p(0x%"PRIxVALUE", 0x%"PRIxVALUE", (const rb_cref_t *)0x%"PRIxVALUE", reg_cfp->ep)) {", ice->flags, ice->value, (VALUE)ice->ic_cref);
fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ice->value);
- fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst);
fprintf(f, " }");
fprintf(f, " else {");
fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size);
diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb
index 0e66f78007..00808b21ff 100644
--- a/tool/ruby_vm/views/mjit_compile.inc.erb
+++ b/tool/ruby_vm/views/mjit_compile.inc.erb
@@ -65,8 +65,8 @@ switch (insn) {
<%= render 'mjit_compile_ivar', locals: { insn: insn } -%>
% when 'invokebuiltin', 'opt_invokebuiltin_delegate'
<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%>
-% when 'opt_getinlinecache'
-<%= render 'mjit_compile_getinlinecache', locals: { insn: insn } -%>
+% when 'opt_getconstant_path'
+<%= render 'mjit_compile_getconstant_path', locals: { insn: insn } -%>
% when 'leave', 'opt_invokebuiltin_delegate_leave'
% # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining.
% if insn.name == 'opt_invokebuiltin_delegate_leave'
diff --git a/vm_core.h b/vm_core.h
index eee25161f5..350f3fdd58 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -256,9 +256,19 @@ STATIC_ASSERT(sizeof_iseq_inline_constant_cache_entry,
struct iseq_inline_constant_cache {
struct iseq_inline_constant_cache_entry *entry;
- // For YJIT: the index to the opt_getinlinecache instruction in the same iseq.
- // It's set during compile time and constant once set.
- unsigned get_insn_idx;
+
+ /**
+ * A null-terminated list of ids, used to represent a constant's path
+ * idNULL is used to represent the :: prefix, and 0 is used to donate the end
+ * of the list.
+ *
+ * For example
+ * FOO {rb_intern("FOO"), 0}
+ * FOO::BAR {rb_intern("FOO"), rb_intern("BAR"), 0}
+ * ::FOO {idNULL, rb_intern("FOO"), 0}
+ * ::FOO::BAR {idNULL, rb_intern("FOO"), rb_intern("BAR"), 0}
+ */
+ const ID *segments;
};
struct iseq_inline_iv_cache_entry {
@@ -339,6 +349,9 @@ typedef uintptr_t iseq_bits_t;
#define ISEQ_IS_SIZE(body) (body->ic_size + body->ivc_size + body->ise_size + body->icvarc_size)
+/* [ TS_IVC | TS_ICVARC | TS_ISE | TS_IC ] */
+#define ISEQ_IS_IC_ENTRY(body, idx) (body->is_entries[(idx) + body->ise_size + body->icvarc_size + body->ivc_size].ic_cache);
+
/* instruction sequence type */
enum rb_iseq_type {
ISEQ_TYPE_TOP,
diff --git a/vm_insnhelper.c b/vm_insnhelper.c
index ab1394c7ca..9ccfdff4a0 100644
--- a/vm_insnhelper.c
+++ b/vm_insnhelper.c
@@ -1040,6 +1040,26 @@ vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_
}
static inline VALUE
+vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
+{
+ VALUE val = Qnil;
+ int idx = 0;
+ int allow_nil = TRUE;
+ if (segments[0] == idNULL) {
+ val = rb_cObject;
+ idx++;
+ allow_nil = FALSE;
+ }
+ while (segments[idx]) {
+ ID id = segments[idx++];
+ val = vm_get_ev_const(ec, val, id, allow_nil, 0);
+ allow_nil = FALSE;
+ }
+ return val;
+}
+
+
+static inline VALUE
vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
{
VALUE klass;
@@ -4949,53 +4969,35 @@ vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr)
#define IMEMO_CONST_CACHE_SHAREABLE IMEMO_FL_USER0
-// This is the iterator used by vm_ic_compile for rb_iseq_each. It is used as a
-// callback for each instruction within the ISEQ, and is meant to return a
-// boolean indicating whether or not to keep iterating.
-//
-// This is used to walk through the ISEQ and find all getconstant instructions
-// between the starting opt_getinlinecache and the ending opt_setinlinecache and
-// associating the inline cache with the constant name components on the VM.
-static bool
-vm_ic_compile_i(VALUE *code, VALUE insn, size_t index, void *ic)
+static void
+vm_track_constant_cache(ID id, void *ic)
{
- if (insn == BIN(opt_setinlinecache)) {
- return false;
- }
+ struct rb_id_table *const_cache = GET_VM()->constant_cache;
+ VALUE lookup_result;
+ st_table *ics;
- if (insn == BIN(getconstant)) {
- ID id = code[index + 1];
- struct rb_id_table *const_cache = GET_VM()->constant_cache;
- VALUE lookup_result;
- st_table *ics;
-
- if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
- ics = (st_table *)lookup_result;
- }
- else {
- ics = st_init_numtable();
- rb_id_table_insert(const_cache, id, (VALUE)ics);
- }
-
- st_insert(ics, (st_data_t) ic, (st_data_t) Qtrue);
+ if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
+ ics = (st_table *)lookup_result;
+ }
+ else {
+ ics = st_init_numtable();
+ rb_id_table_insert(const_cache, id, (VALUE)ics);
}
- return true;
+ st_insert(ics, (st_data_t) ic, (st_data_t) Qtrue);
}
-// Loop through the instruction sequences starting at the opt_getinlinecache
-// call and gather up every getconstant's ID. Associate that with the VM's
-// constant cache so that whenever one of the constants changes the inline cache
-// will get busted.
static void
-vm_ic_compile(rb_control_frame_t *cfp, IC ic)
+vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
{
- const rb_iseq_t *iseq = cfp->iseq;
-
RB_VM_LOCK_ENTER();
- {
- rb_iseq_each(iseq, cfp->pc - ISEQ_BODY(iseq)->iseq_encoded, vm_ic_compile_i, (void *) ic);
+
+ for (int i = 0; segments[i]; i++) {
+ ID id = segments[i];
+ if (id == idNULL) continue;
+ vm_track_constant_cache(id, ic);
}
+
RB_VM_LOCK_LEAVE();
}
@@ -5027,7 +5029,7 @@ rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
}
static void
-vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep)
+vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
{
if (ruby_vm_const_missing_count > 0) {
ruby_vm_const_missing_count = 0;
@@ -5043,7 +5045,9 @@ vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep)
#ifndef MJIT_HEADER
// MJIT and YJIT can't be on at the same time, so there is no need to
// notify YJIT about changes to the IC when running inside MJIT code.
- rb_yjit_constant_ic_update(iseq, ic);
+ RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
+ unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
+ rb_yjit_constant_ic_update(iseq, ic, pos);
#endif
}
diff --git a/yjit.h b/yjit.h
index c9dc52c9b6..7884aef18d 100644
--- a/yjit.h
+++ b/yjit.h
@@ -41,7 +41,7 @@ void rb_yjit_iseq_mark(void *payload);
void rb_yjit_iseq_update_references(void *payload);
void rb_yjit_iseq_free(void *payload);
void rb_yjit_before_ractor_spawn(void);
-void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic);
+void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx);
void rb_yjit_tracing_invalidate_all(void);
#else
@@ -64,7 +64,7 @@ static inline void rb_yjit_iseq_mark(void *payload) {}
static inline void rb_yjit_iseq_update_references(void *payload) {}
static inline void rb_yjit_iseq_free(void *payload) {}
static inline void rb_yjit_before_ractor_spawn(void) {}
-static inline void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic) {}
+static inline void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx) {}
static inline void rb_yjit_tracing_invalidate_all(void) {}
#endif // #if USE_YJIT
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index ac42b4a1e2..58b30d5c85 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -769,9 +769,10 @@ pub fn gen_single_block(
.try_into()
.unwrap();
- // opt_getinlinecache wants to be in a block all on its own. Cut the block short
- // if we run into it. See gen_opt_getinlinecache() for details.
- if opcode == YARVINSN_opt_getinlinecache.as_usize() && insn_idx > starting_insn_idx {
+ // We need opt_getconstant_path to be in a block all on its own. Cut the block short
+ // if we run into it. This is necessary because we want to invalidate based on the
+ // instruction's index.
+ if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > starting_insn_idx {
jump_to_next_insn(&mut jit, &ctx, &mut asm, ocb);
break;
}
@@ -5508,15 +5509,15 @@ fn gen_setclassvariable(
KeepCompiling
}
-fn gen_opt_getinlinecache(
+fn gen_opt_getconstant_path(
jit: &mut JITState,
ctx: &mut Context,
asm: &mut Assembler,
ocb: &mut OutlinedCb,
) -> CodegenStatus {
- let jump_offset = jit_get_arg(jit, 0);
- let const_cache_as_value = jit_get_arg(jit, 1);
+ let const_cache_as_value = jit_get_arg(jit, 0);
let ic: *const iseq_inline_constant_cache = const_cache_as_value.as_ptr();
+ let idlist: *const ID = unsafe { (*ic).segments };
// See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update().
let ice = unsafe { (*ic).entry };
@@ -5573,22 +5574,12 @@ fn gen_opt_getinlinecache(
// Invalidate output code on any constant writes associated with
// constants referenced within the current block.
- assume_stable_constant_names(jit, ocb);
+ assume_stable_constant_names(jit, ocb, idlist);
jit_putobject(jit, ctx, asm, unsafe { (*ice).value });
}
- // Jump over the code for filling the cache
- let jump_idx = jit_next_insn_idx(jit) + jump_offset.as_u32();
- gen_direct_jump(
- jit,
- ctx,
- BlockId {
- iseq: jit.iseq,
- idx: jump_idx,
- },
- asm,
- );
+ jump_to_next_insn(jit, ctx, asm, ocb);
EndBlock
}
@@ -5920,7 +5911,7 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> {
YARVINSN_opt_size => Some(gen_opt_size),
YARVINSN_opt_length => Some(gen_opt_length),
YARVINSN_opt_regexpmatch2 => Some(gen_opt_regexpmatch2),
- YARVINSN_opt_getinlinecache => Some(gen_opt_getinlinecache),
+ YARVINSN_opt_getconstant_path => Some(gen_opt_getconstant_path),
YARVINSN_invokebuiltin => Some(gen_invokebuiltin),
YARVINSN_opt_invokebuiltin_delegate => Some(gen_opt_invokebuiltin_delegate),
YARVINSN_opt_invokebuiltin_delegate_leave => Some(gen_opt_invokebuiltin_delegate),
diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs
index 591408e1da..cedc3216b8 100644
--- a/yjit/src/cruby_bindings.inc.rs
+++ b/yjit/src/cruby_bindings.inc.rs
@@ -588,7 +588,7 @@ pub struct iseq_inline_constant_cache_entry {
#[derive(Debug, Copy, Clone)]
pub struct iseq_inline_constant_cache {
pub entry: *mut iseq_inline_constant_cache_entry,
- pub get_insn_idx: ::std::os::raw::c_uint,
+ pub segments: *const ID,
}
#[repr(C)]
#[derive(Debug, Copy, Clone)]
@@ -784,108 +784,108 @@ pub const YARVINSN_getinstancevariable: ruby_vminsn_type = 8;
pub const YARVINSN_setinstancevariable: ruby_vminsn_type = 9;
pub const YARVINSN_getclassvariable: ruby_vminsn_type = 10;
pub const YARVINSN_setclassvariable: ruby_vminsn_type = 11;
-pub const YARVINSN_getconstant: ruby_vminsn_type = 12;
-pub const YARVINSN_setconstant: ruby_vminsn_type = 13;
-pub const YARVINSN_getglobal: ruby_vminsn_type = 14;
-pub const YARVINSN_setglobal: ruby_vminsn_type = 15;
-pub const YARVINSN_putnil: ruby_vminsn_type = 16;
-pub const YARVINSN_putself: ruby_vminsn_type = 17;
-pub const YARVINSN_putobject: ruby_vminsn_type = 18;
-pub const YARVINSN_putspecialobject: ruby_vminsn_type = 19;
-pub const YARVINSN_putstring: ruby_vminsn_type = 20;
-pub const YARVINSN_concatstrings: ruby_vminsn_type = 21;
-pub const YARVINSN_anytostring: ruby_vminsn_type = 22;
-pub const YARVINSN_toregexp: ruby_vminsn_type = 23;
-pub const YARVINSN_intern: ruby_vminsn_type = 24;
-pub const YARVINSN_newarray: ruby_vminsn_type = 25;
-pub const YARVINSN_newarraykwsplat: ruby_vminsn_type = 26;
-pub const YARVINSN_duparray: ruby_vminsn_type = 27;
-pub const YARVINSN_duphash: ruby_vminsn_type = 28;
-pub const YARVINSN_expandarray: ruby_vminsn_type = 29;
-pub const YARVINSN_concatarray: ruby_vminsn_type = 30;
-pub const YARVINSN_splatarray: ruby_vminsn_type = 31;
-pub const YARVINSN_newhash: ruby_vminsn_type = 32;
-pub const YARVINSN_newrange: ruby_vminsn_type = 33;
-pub const YARVINSN_pop: ruby_vminsn_type = 34;
-pub const YARVINSN_dup: ruby_vminsn_type = 35;
-pub const YARVINSN_dupn: ruby_vminsn_type = 36;
-pub const YARVINSN_swap: ruby_vminsn_type = 37;
-pub const YARVINSN_opt_reverse: ruby_vminsn_type = 38;
-pub const YARVINSN_topn: ruby_vminsn_type = 39;
-pub const YARVINSN_setn: ruby_vminsn_type = 40;
-pub const YARVINSN_adjuststack: ruby_vminsn_type = 41;
-pub const YARVINSN_defined: ruby_vminsn_type = 42;
-pub const YARVINSN_checkmatch: ruby_vminsn_type = 43;
-pub const YARVINSN_checkkeyword: ruby_vminsn_type = 44;
-pub const YARVINSN_checktype: ruby_vminsn_type = 45;
-pub const YARVINSN_defineclass: ruby_vminsn_type = 46;
-pub const YARVINSN_definemethod: ruby_vminsn_type = 47;
-pub const YARVINSN_definesmethod: ruby_vminsn_type = 48;
-pub const YARVINSN_send: ruby_vminsn_type = 49;
-pub const YARVINSN_opt_send_without_block: ruby_vminsn_type = 50;
-pub const YARVINSN_objtostring: ruby_vminsn_type = 51;
-pub const YARVINSN_opt_str_freeze: ruby_vminsn_type = 52;
-pub const YARVINSN_opt_nil_p: ruby_vminsn_type = 53;
-pub const YARVINSN_opt_str_uminus: ruby_vminsn_type = 54;
-pub const YARVINSN_opt_newarray_max: ruby_vminsn_type = 55;
-pub const YARVINSN_opt_newarray_min: ruby_vminsn_type = 56;
-pub const YARVINSN_invokesuper: ruby_vminsn_type = 57;
-pub const YARVINSN_invokeblock: ruby_vminsn_type = 58;
-pub const YARVINSN_leave: ruby_vminsn_type = 59;
-pub const YARVINSN_throw: ruby_vminsn_type = 60;
-pub const YARVINSN_jump: ruby_vminsn_type = 61;
-pub const YARVINSN_branchif: ruby_vminsn_type = 62;
-pub const YARVINSN_branchunless: ruby_vminsn_type = 63;
-pub const YARVINSN_branchnil: ruby_vminsn_type = 64;
-pub const YARVINSN_opt_getinlinecache: ruby_vminsn_type = 65;
-pub const YARVINSN_opt_setinlinecache: ruby_vminsn_type = 66;
-pub const YARVINSN_once: ruby_vminsn_type = 67;
-pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 68;
-pub const YARVINSN_opt_plus: ruby_vminsn_type = 69;
-pub const YARVINSN_opt_minus: ruby_vminsn_type = 70;
-pub const YARVINSN_opt_mult: ruby_vminsn_type = 71;
-pub const YARVINSN_opt_div: ruby_vminsn_type = 72;
-pub const YARVINSN_opt_mod: ruby_vminsn_type = 73;
-pub const YARVINSN_opt_eq: ruby_vminsn_type = 74;
-pub const YARVINSN_opt_neq: ruby_vminsn_type = 75;
-pub const YARVINSN_opt_lt: ruby_vminsn_type = 76;
-pub const YARVINSN_opt_le: ruby_vminsn_type = 77;
-pub const YARVINSN_opt_gt: ruby_vminsn_type = 78;
-pub const YARVINSN_opt_ge: ruby_vminsn_type = 79;
-pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 80;
-pub const YARVINSN_opt_and: ruby_vminsn_type = 81;
-pub const YARVINSN_opt_or: ruby_vminsn_type = 82;
-pub const YARVINSN_opt_aref: ruby_vminsn_type = 83;
-pub const YARVINSN_opt_aset: ruby_vminsn_type = 84;
-pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 85;
-pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 86;
-pub const YARVINSN_opt_length: ruby_vminsn_type = 87;
-pub const YARVINSN_opt_size: ruby_vminsn_type = 88;
-pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 89;
-pub const YARVINSN_opt_succ: ruby_vminsn_type = 90;
-pub const YARVINSN_opt_not: ruby_vminsn_type = 91;
-pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 92;
-pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 93;
-pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 94;
-pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 95;
-pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 96;
-pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 97;
-pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 98;
-pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 99;
-pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 100;
-pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 101;
-pub const YARVINSN_trace_nop: ruby_vminsn_type = 102;
-pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 103;
-pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 104;
-pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 105;
-pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 106;
-pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 107;
-pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 108;
-pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 109;
-pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 110;
-pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 111;
-pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 112;
-pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 113;
+pub const YARVINSN_opt_getconstant_path: ruby_vminsn_type = 12;
+pub const YARVINSN_getconstant: ruby_vminsn_type = 13;
+pub const YARVINSN_setconstant: ruby_vminsn_type = 14;
+pub const YARVINSN_getglobal: ruby_vminsn_type = 15;
+pub const YARVINSN_setglobal: ruby_vminsn_type = 16;
+pub const YARVINSN_putnil: ruby_vminsn_type = 17;
+pub const YARVINSN_putself: ruby_vminsn_type = 18;
+pub const YARVINSN_putobject: ruby_vminsn_type = 19;
+pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20;
+pub const YARVINSN_putstring: ruby_vminsn_type = 21;
+pub const YARVINSN_concatstrings: ruby_vminsn_type = 22;
+pub const YARVINSN_anytostring: ruby_vminsn_type = 23;
+pub const YARVINSN_toregexp: ruby_vminsn_type = 24;
+pub const YARVINSN_intern: ruby_vminsn_type = 25;
+pub const YARVINSN_newarray: ruby_vminsn_type = 26;
+pub const YARVINSN_newarraykwsplat: ruby_vminsn_type = 27;
+pub const YARVINSN_duparray: ruby_vminsn_type = 28;
+pub const YARVINSN_duphash: ruby_vminsn_type = 29;
+pub const YARVINSN_expandarray: ruby_vminsn_type = 30;
+pub const YARVINSN_concatarray: ruby_vminsn_type = 31;
+pub const YARVINSN_splatarray: ruby_vminsn_type = 32;
+pub const YARVINSN_newhash: ruby_vminsn_type = 33;
+pub const YARVINSN_newrange: ruby_vminsn_type = 34;
+pub const YARVINSN_pop: ruby_vminsn_type = 35;
+pub const YARVINSN_dup: ruby_vminsn_type = 36;
+pub const YARVINSN_dupn: ruby_vminsn_type = 37;
+pub const YARVINSN_swap: ruby_vminsn_type = 38;
+pub const YARVINSN_opt_reverse: ruby_vminsn_type = 39;
+pub const YARVINSN_topn: ruby_vminsn_type = 40;
+pub const YARVINSN_setn: ruby_vminsn_type = 41;
+pub const YARVINSN_adjuststack: ruby_vminsn_type = 42;
+pub const YARVINSN_defined: ruby_vminsn_type = 43;
+pub const YARVINSN_checkmatch: ruby_vminsn_type = 44;
+pub const YARVINSN_checkkeyword: ruby_vminsn_type = 45;
+pub const YARVINSN_checktype: ruby_vminsn_type = 46;
+pub const YARVINSN_defineclass: ruby_vminsn_type = 47;
+pub const YARVINSN_definemethod: ruby_vminsn_type = 48;
+pub const YARVINSN_definesmethod: ruby_vminsn_type = 49;
+pub const YARVINSN_send: ruby_vminsn_type = 50;
+pub const YARVINSN_opt_send_without_block: ruby_vminsn_type = 51;
+pub const YARVINSN_objtostring: ruby_vminsn_type = 52;
+pub const YARVINSN_opt_str_freeze: ruby_vminsn_type = 53;
+pub const YARVINSN_opt_nil_p: ruby_vminsn_type = 54;
+pub const YARVINSN_opt_str_uminus: ruby_vminsn_type = 55;
+pub const YARVINSN_opt_newarray_max: ruby_vminsn_type = 56;
+pub const YARVINSN_opt_newarray_min: ruby_vminsn_type = 57;
+pub const YARVINSN_invokesuper: ruby_vminsn_type = 58;
+pub const YARVINSN_invokeblock: ruby_vminsn_type = 59;
+pub const YARVINSN_leave: ruby_vminsn_type = 60;
+pub const YARVINSN_throw: ruby_vminsn_type = 61;
+pub const YARVINSN_jump: ruby_vminsn_type = 62;
+pub const YARVINSN_branchif: ruby_vminsn_type = 63;
+pub const YARVINSN_branchunless: ruby_vminsn_type = 64;
+pub const YARVINSN_branchnil: ruby_vminsn_type = 65;
+pub const YARVINSN_once: ruby_vminsn_type = 66;
+pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 67;
+pub const YARVINSN_opt_plus: ruby_vminsn_type = 68;
+pub const YARVINSN_opt_minus: ruby_vminsn_type = 69;
+pub const YARVINSN_opt_mult: ruby_vminsn_type = 70;
+pub const YARVINSN_opt_div: ruby_vminsn_type = 71;
+pub const YARVINSN_opt_mod: ruby_vminsn_type = 72;
+pub const YARVINSN_opt_eq: ruby_vminsn_type = 73;
+pub const YARVINSN_opt_neq: ruby_vminsn_type = 74;
+pub const YARVINSN_opt_lt: ruby_vminsn_type = 75;
+pub const YARVINSN_opt_le: ruby_vminsn_type = 76;
+pub const YARVINSN_opt_gt: ruby_vminsn_type = 77;
+pub const YARVINSN_opt_ge: ruby_vminsn_type = 78;
+pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 79;
+pub const YARVINSN_opt_and: ruby_vminsn_type = 80;
+pub const YARVINSN_opt_or: ruby_vminsn_type = 81;
+pub const YARVINSN_opt_aref: ruby_vminsn_type = 82;
+pub const YARVINSN_opt_aset: ruby_vminsn_type = 83;
+pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 84;
+pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 85;
+pub const YARVINSN_opt_length: ruby_vminsn_type = 86;
+pub const YARVINSN_opt_size: ruby_vminsn_type = 87;
+pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 88;
+pub const YARVINSN_opt_succ: ruby_vminsn_type = 89;
+pub const YARVINSN_opt_not: ruby_vminsn_type = 90;
+pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 91;
+pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 92;
+pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 93;
+pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 94;
+pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 95;
+pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 96;
+pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 97;
+pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 98;
+pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 99;
+pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 100;
+pub const YARVINSN_trace_nop: ruby_vminsn_type = 101;
+pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 102;
+pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 103;
+pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 104;
+pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 105;
+pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 106;
+pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 107;
+pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 108;
+pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 109;
+pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 110;
+pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 111;
+pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 112;
+pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 113;
pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 114;
pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 115;
pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 116;
@@ -939,44 +939,42 @@ pub const YARVINSN_trace_jump: ruby_vminsn_type = 163;
pub const YARVINSN_trace_branchif: ruby_vminsn_type = 164;
pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 165;
pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 166;
-pub const YARVINSN_trace_opt_getinlinecache: ruby_vminsn_type = 167;
-pub const YARVINSN_trace_opt_setinlinecache: ruby_vminsn_type = 168;
-pub const YARVINSN_trace_once: ruby_vminsn_type = 169;
-pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 170;
-pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 171;
-pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 172;
-pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 173;
-pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 174;
-pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 175;
-pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 176;
-pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 177;
-pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 178;
-pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 179;
-pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 180;
-pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 181;
-pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 182;
-pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 183;
-pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 184;
-pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 185;
-pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 186;
-pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 187;
-pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 188;
-pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 189;
-pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 190;
-pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 191;
-pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 192;
-pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 193;
-pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 194;
-pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 195;
-pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 196;
-pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 197;
-pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 198;
-pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 199;
-pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 200;
-pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 201;
-pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 202;
-pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 203;
-pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 204;
+pub const YARVINSN_trace_once: ruby_vminsn_type = 167;
+pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 168;
+pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 169;
+pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 170;
+pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 171;
+pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 172;
+pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 173;
+pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 174;
+pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 175;
+pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 176;
+pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 177;
+pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 178;
+pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 179;
+pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 180;
+pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 181;
+pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 182;
+pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 183;
+pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 184;
+pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 185;
+pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 186;
+pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 187;
+pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 188;
+pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 189;
+pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 190;
+pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 191;
+pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 192;
+pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 193;
+pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 194;
+pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 195;
+pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 196;
+pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 197;
+pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 198;
+pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 199;
+pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 200;
+pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 201;
+pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 202;
pub type ruby_vminsn_type = u32;
extern "C" {
pub fn rb_vm_insn_addr2opcode(addr: *const ::std::os::raw::c_void) -> ::std::os::raw::c_int;
diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs
index 9cdef0d8bb..4ed21118cc 100644
--- a/yjit/src/invariants.rs
+++ b/yjit/src/invariants.rs
@@ -12,7 +12,6 @@ use crate::yjit::yjit_enabled_p;
use std::collections::{HashMap, HashSet};
use std::mem;
-use std::os::raw::c_void;
// Invariants to track:
// assume_bop_not_redefined(jit, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS)
@@ -173,57 +172,41 @@ pub fn assume_single_ractor_mode(jit: &mut JITState, ocb: &mut OutlinedCb) -> bo
/// subsequent opt_setinlinecache and find all of the name components that are
/// associated with this constant (which correspond to the getconstant
/// arguments).
-pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb) {
+pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, idlist: *const ID) {
/// Tracks that a block is assuming that the name component of a constant
/// has not changed since the last call to this function.
- unsafe extern "C" fn assume_stable_constant_name(
- code: *mut VALUE,
- insn: VALUE,
- index: u64,
- data: *mut c_void,
- ) -> bool {
- if insn.as_u32() == YARVINSN_opt_setinlinecache {
- return false;
+ fn assume_stable_constant_name(
+ jit: &mut JITState,
+ id: ID,
+ ) {
+ if id == idNULL as u64 {
+ // Used for :: prefix
+ return;
}
- if insn.as_u32() == YARVINSN_getconstant {
- let jit = &mut *(data as *mut JITState);
+ let invariants = Invariants::get_instance();
+ invariants
+ .constant_state_blocks
+ .entry(id)
+ .or_default()
+ .insert(jit.get_block());
+ invariants
+ .block_constant_states
+ .entry(jit.get_block())
+ .or_default()
+ .insert(id);
+ }
- // The first operand to GETCONSTANT is always the ID associated with
- // the constant lookup. We are grabbing this out in order to
- // associate this block with the stability of this constant name.
- let id = code.add(index.as_usize() + 1).read().as_u64() as ID;
- let invariants = Invariants::get_instance();
- invariants
- .constant_state_blocks
- .entry(id)
- .or_default()
- .insert(jit.get_block());
- invariants
- .block_constant_states
- .entry(jit.get_block())
- .or_default()
- .insert(id);
+ for i in 0.. {
+ match unsafe { *idlist.offset(i) } {
+ 0 => break, // End of NULL terminated list
+ id => assume_stable_constant_name(jit, id),
}
-
- true
}
jit_ensure_block_entry_exit(jit, ocb);
- unsafe {
- let iseq = jit.get_iseq();
- let encoded = get_iseq_body_iseq_encoded(iseq);
- let start_index = jit.get_pc().offset_from(encoded);
-
- rb_iseq_each(
- iseq,
- start_index.try_into().unwrap(),
- Some(assume_stable_constant_name),
- jit as *mut _ as *mut c_void,
- );
- };
}
/// Called when a basic operator is redefined. Note that all the blocks assuming
@@ -450,7 +433,7 @@ pub fn block_assumptions_free(blockref: &BlockRef) {
/// Invalidate the block for the matching opt_getinlinecache so it could regenerate code
/// using the new value in the constant cache.
#[no_mangle]
-pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC) {
+pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: u32) {
// If YJIT isn't enabled, do nothing
if !yjit_enabled_p() {
return;
@@ -464,34 +447,33 @@ pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC) {
with_vm_lock(src_loc!(), || {
let code = unsafe { get_iseq_body_iseq_encoded(iseq) };
- let get_insn_idx = unsafe { (*ic).get_insn_idx };
// This should come from a running iseq, so direct threading translation
// should have been done
assert!(unsafe { FL_TEST(iseq.into(), VALUE(ISEQ_TRANSLATED as usize)) } != VALUE(0));
- assert!(get_insn_idx < unsafe { get_iseq_encoded_size(iseq) });
+ assert!(insn_idx < unsafe { get_iseq_encoded_size(iseq) });
- // Ensure that the instruction the get_insn_idx is pointing to is in
- // fact a opt_getinlinecache instruction.
+ // Ensure that the instruction the insn_idx is pointing to is in
+ // fact a opt_getconstant_path instruction.
assert_eq!(
unsafe {
- let opcode_pc = code.add(get_insn_idx.as_usize());
+ let opcode_pc = code.add(insn_idx.as_usize());
let translated_opcode: VALUE = opcode_pc.read();
rb_vm_insn_decode(translated_opcode)
},
- YARVINSN_opt_getinlinecache.try_into().unwrap()
+ YARVINSN_opt_getconstant_path.try_into().unwrap()
);
// Find the matching opt_getinlinecache and invalidate all the blocks there
// RUBY_ASSERT(insn_op_type(BIN(opt_getinlinecache), 1) == TS_IC);
- let ic_pc = unsafe { code.add(get_insn_idx.as_usize() + 2) };
+ let ic_pc = unsafe { code.add(insn_idx.as_usize() + 1) };
let ic_operand: IC = unsafe { ic_pc.read() }.as_mut_ptr();
if ic == ic_operand {
for block in take_version_list(BlockId {
iseq,
- idx: get_insn_idx,
+ idx: insn_idx,
}) {
invalidate_block_version(&block);
incr_counter!(invalidate_constant_ic_fill);
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-13 03:25:39 +0000