Filling cache values on cvar write

Instead of on read. Once it's in the inline cache we never have to make
one again. We want to eventually put the value into the cache, and the
best opportunity to do that is when you write the value.

6 files changed, 65 insertions, 12 deletions

diff --git a/class.c b/class.c
index 07235b80d2..ef3db6dab3 100644
--- a/class.c
+++ b/class.c
@@ -960,6 +960,7 @@ rb_include_class_new(VALUE module, VALUE super)
 	RCLASS_CONST_TBL(module) = rb_id_table_create(0);
     }
     RCLASS_IV_TBL(klass) = RCLASS_IV_TBL(module);
+    RCLASS_CVC_TBL(klass) = RCLASS_CVC_TBL(module);
     RCLASS_CONST_TBL(klass) = RCLASS_CONST_TBL(module);
 
     RCLASS_SET_SUPER(klass, super);
diff --git a/compile.c b/compile.c
index 804be4bffb..fa6af03c9c 100644
--- a/compile.c
+++ b/compile.c
@@ -8044,8 +8044,9 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, in
 	if (!popped) {
 	    ADD_INSN(ret, line_node, dup);
 	}
-	ADD_INSN1(ret, line_node, setclassvariable,
-		  ID2SYM(node->nd_vid));
+        ADD_INSN2(ret, line_node, setclassvariable,
+                  ID2SYM(node->nd_vid),
+                  get_ivar_ic_value(iseq,node->nd_vid));
 	break;
       }
       case NODE_OP_ASGN1: {
diff --git a/debug_counter.h b/debug_counter.h
index 97a758ca94..3cf80cc188 100644
--- a/debug_counter.h
+++ b/debug_counter.h
@@ -24,7 +24,8 @@ RB_DEBUG_COUNTER(mc_inline_miss_same_cme)    // IMC miss, but same CME
 RB_DEBUG_COUNTER(mc_inline_miss_same_def)    // IMC miss, but same definition
 RB_DEBUG_COUNTER(mc_inline_miss_diff)        // IMC miss, different methods
 
-RB_DEBUG_COUNTER(cvar_inline_hit)            // cvar cache hit
+RB_DEBUG_COUNTER(cvar_write_inline_hit)      // cvar cache hit on write
+RB_DEBUG_COUNTER(cvar_read_inline_hit)       // cvar cache hit on read
 RB_DEBUG_COUNTER(cvar_inline_miss)           // miss inline cache
 RB_DEBUG_COUNTER(cvar_class_invalidate)      // invalidate cvar cache when define a cvar that's defined on a subclass
 RB_DEBUG_COUNTER(cvar_include_invalidate)    // invalidate cvar cache on module include or prepend
diff --git a/insns.def b/insns.def
index a565c1123f..e1e4577cd9 100644
--- a/insns.def
+++ b/insns.def
@@ -244,14 +244,14 @@ getclassvariable
 /* Set value of class variable id of klass as val. */
 DEFINE_INSN
 setclassvariable
-(ID id)
+(ID id, IVC ic)
 (VALUE val)
 ()
 /* "class variable access from toplevel" warning can be hooked. */
 // attr bool leaf = false; /* has rb_warning() */
 {
     vm_ensure_not_refinement_module(GET_SELF());
-    vm_setclassvariable(vm_get_cref(GET_EP()), GET_CFP(), id,  val);
+    vm_setclassvariable(GET_ISEQ(), vm_get_cref(GET_EP()), GET_CFP(), id,  val, (ICVARC)ic);
 }
 
 /* Get constant variable id. If klass is Qnil and allow_nil is Qtrue, constants
diff --git a/variable.c b/variable.c
index 82918037f9..8c29fbdf3b 100644
--- a/variable.c
+++ b/variable.c
@@ -40,6 +40,7 @@
 #include "vm_sync.h"
 
 RUBY_EXTERN rb_serial_t ruby_vm_global_cvar_state;
+#define GET_GLOBAL_CVAR_STATE() (ruby_vm_global_cvar_state)
 
 typedef void rb_gvar_compact_t(void *var);
 
@@ -3399,6 +3400,24 @@ rb_cvar_set(VALUE klass, ID id, VALUE val)
 
     int result = rb_class_ivar_set(target, id, val);
 
+    struct rb_id_table *rb_cvc_tbl = RCLASS_CVC_TBL(target);
+
+    if (!rb_cvc_tbl) {
+        rb_cvc_tbl = RCLASS_CVC_TBL(target) = rb_id_table_create(2);
+    }
+
+    struct rb_cvar_class_tbl_entry *ent;
+
+    if (!rb_id_table_lookup(rb_cvc_tbl, id, (VALUE*)&ent)) {
+        ent = ALLOC(struct rb_cvar_class_tbl_entry);
+        ent->class_value = target;
+        ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
+        rb_id_table_insert(rb_cvc_tbl, id, (VALUE)ent);
+        RB_DEBUG_COUNTER_INC(cvar_inline_miss);
+    } else {
+        ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
+    }
+
     // Break the cvar cache if this is a new class variable
     // and target is a module or a subclass with the same
     // cvar in this lookup.
diff --git a/vm_insnhelper.c b/vm_insnhelper.c
index 0aeab30077..72c90a8a5c 100644
--- a/vm_insnhelper.c
+++ b/vm_insnhelper.c
@@ -1286,7 +1286,7 @@ vm_getclassvariable(const rb_iseq_t *iseq, const rb_cref_t *cref, const rb_contr
 {
     if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE()) {
         VALUE v = Qundef;
-        RB_DEBUG_COUNTER_INC(cvar_inline_hit);
+        RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
 
         if (st_lookup(RCLASS_IV_TBL(ic->entry->class_value), (st_data_t)id, &v)) {
             return v;
@@ -1298,6 +1298,10 @@ vm_getclassvariable(const rb_iseq_t *iseq, const rb_cref_t *cref, const rb_contr
 
     VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
 
+    if (RB_TYPE_P(defined_class, T_ICLASS)) {
+        defined_class = RBASIC(defined_class)->klass;
+    }
+
     struct rb_id_table *rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
 
     if (!rb_cvc_tbl) {
@@ -1307,11 +1311,7 @@ vm_getclassvariable(const rb_iseq_t *iseq, const rb_cref_t *cref, const rb_contr
     struct rb_cvar_class_tbl_entry *ent;
 
     if (!rb_id_table_lookup(rb_cvc_tbl, id, (VALUE*)&ent)) {
-        ent = ALLOC(struct rb_cvar_class_tbl_entry);
-        ent->class_value = defined_class;
-        ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
-        rb_id_table_insert(rb_cvc_tbl, id, (VALUE)ent);
-        RB_DEBUG_COUNTER_INC(cvar_inline_miss);
+        rb_bug("should have cvar cache entry");
     } else {
         ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
     }
@@ -1323,11 +1323,42 @@ vm_getclassvariable(const rb_iseq_t *iseq, const rb_cref_t *cref, const rb_contr
 }
 
 static inline void
-vm_setclassvariable(const rb_cref_t *cref, const rb_control_frame_t *cfp, ID id, VALUE val)
+vm_setclassvariable(const rb_iseq_t *iseq, const rb_cref_t *cref, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
 {
+    if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE()) {
+        RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
+
+        rb_class_ivar_set(ic->entry->class_value, id, val);
+        return;
+    }
+
     VALUE klass = vm_get_cvar_base(cref, cfp, 1);
 
     rb_cvar_set(klass, id, val);
+
+    VALUE defined_class = 0;
+    rb_cvar_find(klass, id, &defined_class);
+
+    if (RB_TYPE_P(defined_class, T_ICLASS)) {
+        defined_class = RBASIC(defined_class)->klass;
+    }
+
+    struct rb_id_table *rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
+
+    if (!rb_cvc_tbl) {
+        rb_bug("the cvc table should be set");
+    }
+
+    struct rb_cvar_class_tbl_entry *ent;
+
+    if (!rb_id_table_lookup(rb_cvc_tbl, id, (VALUE*)&ent)) {
+        rb_bug("should have cvar cache entry");
+    } else {
+        ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
+    }
+
+    ic->entry = ent;
+    RB_OBJ_WRITTEN(iseq, Qundef, ent->class_value);
 }
 
 static inline VALUE