1 files changed, 80 insertions, 61 deletions

diff --git a/internal/imemo.h b/internal/imemo.h
index 8d1e42353d..e8a5f0fc8e 100644
--- a/internal/imemo.h
+++ b/internal/imemo.h
@@ -37,10 +37,13 @@ enum imemo_type {
     imemo_ment           =  6,
     imemo_iseq           =  7,
     imemo_tmpbuf         =  8,
+    imemo_cvar_entry     =  9,
     imemo_callinfo       = 10,
     imemo_callcache      = 11,
     imemo_constcache     = 12,
-    imemo_fields   = 13,
+    imemo_fields         = 13,
+    imemo_subclasses     = 14,
+    imemo_cdhash         = 15,
 };
 
 /* CREF (Class REFerence) is defined in method.h */
@@ -57,7 +60,6 @@ struct vm_svar {
 /*! THROW_DATA */
 struct vm_throw_data {
     VALUE flags;
-    VALUE reserved;
     const VALUE throw_obj;
     const struct rb_control_frame_struct *catch_frame;
     int throw_state;
@@ -93,30 +95,36 @@ struct vm_ifunc {
 
 struct rb_imemo_tmpbuf_struct {
     VALUE flags;
-    VALUE reserved;
     VALUE *ptr; /* malloc'ed buffer */
-    struct rb_imemo_tmpbuf_struct *next; /* next imemo */
-    size_t cnt; /* buffer size in VALUE */
+    size_t size; /* buffer size in bytes */
 };
 
+struct rb_imemo_cdhash {
+    VALUE flags;
+    st_table tbl;
+};
+
+/* Set on imemo_memo when u3 holds a VALUE that GC must mark.
+ * When unset, u3 is a non-VALUE (cnt/state). */
+#define MEMO_U3_IS_VALUE IMEMO_FL_USER0
+
 /*! MEMO
  *
  * @see imemo_type
  * */
 struct MEMO {
     VALUE flags;
-    VALUE reserved;
     const VALUE v1;
     const VALUE v2;
     union {
         long cnt;
         long state;
         const VALUE value;
-        void (*func)(void);
     } u3;
 };
 
-#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T)))
+#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), false))
+#define SHAREABLE_IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), true))
 
 /* ment is in method.h */
 
@@ -133,16 +141,16 @@ struct MEMO {
 #ifndef RUBY_RUBYPARSER_H
 typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t;
 #endif
-rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
+VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size, bool is_shareable);
+VALUE rb_imemo_tmpbuf_new(void);
+struct MEMO *rb_imemo_memo_new(VALUE a, VALUE b, long c);
+struct MEMO *rb_imemo_memo_new_value(VALUE a, VALUE b, VALUE c);
 struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc);
 static inline enum imemo_type imemo_type(VALUE imemo);
 static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type);
 static inline bool imemo_throw_data_p(VALUE imemo);
 static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data);
-static inline VALUE rb_imemo_tmpbuf_auto_free_pointer(void);
 static inline void *RB_IMEMO_TMPBUF_PTR(VALUE v);
-static inline void *rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr);
-static inline VALUE rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str);
 static inline void MEMO_V1_SET(struct MEMO *m, VALUE v);
 static inline void MEMO_V2_SET(struct MEMO *m, VALUE v);
 
@@ -151,21 +159,9 @@ void rb_imemo_mark_and_move(VALUE obj, bool reference_updating);
 void rb_imemo_free(VALUE obj);
 
 RUBY_SYMBOL_EXPORT_BEGIN
-VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size);
 const char *rb_imemo_name(enum imemo_type type);
 RUBY_SYMBOL_EXPORT_END
 
-static inline struct MEMO *
-MEMO_NEW(VALUE a, VALUE b, VALUE c)
-{
-    struct MEMO *memo = IMEMO_NEW(struct MEMO, imemo_memo, 0);
-    *((VALUE *)&memo->v1) = a;
-    *((VALUE *)&memo->v2) = b;
-    *((VALUE *)&memo->u3.value) = c;
-
-    return memo;
-}
-
 static inline enum imemo_type
 imemo_type(VALUE imemo)
 {
@@ -201,12 +197,6 @@ rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data)
     return rb_vm_ifunc_new(func, data, 0, UNLIMITED_ARGUMENTS);
 }
 
-static inline VALUE
-rb_imemo_tmpbuf_auto_free_pointer(void)
-{
-    return rb_imemo_new(imemo_tmpbuf, 0, sizeof(rb_imemo_tmpbuf_t));
-}
-
 static inline void *
 RB_IMEMO_TMPBUF_PTR(VALUE v)
 {
@@ -214,30 +204,16 @@ RB_IMEMO_TMPBUF_PTR(VALUE v)
     return p->ptr;
 }
 
-static inline void *
-rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr)
-{
-    return ((rb_imemo_tmpbuf_t *)v)->ptr = ptr;
-}
-
 static inline VALUE
-rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str)
+rb_imemo_tmpbuf_new_from_an_RString(VALUE str)
 {
-    const void *src;
     VALUE imemo;
-    rb_imemo_tmpbuf_t *tmpbuf;
-    void *dst;
     size_t len;
 
     StringValue(str);
-    /* create tmpbuf to keep the pointer before xmalloc */
-    imemo = rb_imemo_tmpbuf_auto_free_pointer();
-    tmpbuf = (rb_imemo_tmpbuf_t *)imemo;
     len = RSTRING_LEN(str);
-    src = RSTRING_PTR(str);
-    dst = ruby_xmalloc(len);
-    memcpy(dst, src, len);
-    tmpbuf->ptr = dst;
+    rb_alloc_tmp_buffer(&imemo, len);
+    memcpy(RB_IMEMO_TMPBUF_PTR(imemo), RSTRING_PTR(str), len);
     return imemo;
 }
 
@@ -253,6 +229,15 @@ MEMO_V2_SET(struct MEMO *m, VALUE v)
     RB_OBJ_WRITE(m, &m->v2, v);
 }
 
+VALUE rb_imemo_cdhash_new(size_t size, const struct st_hash_type *type);
+
+static inline st_table *
+rb_imemo_cdhash_tbl(VALUE obj)
+{
+    RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_cdhash));
+    return &((struct rb_imemo_cdhash *)obj)->tbl;
+}
+
 struct rb_fields {
     struct RBasic basic;
     union {
@@ -270,48 +255,82 @@ struct rb_fields {
     } as;
 };
 
-#define OBJ_FIELD_EXTERNAL IMEMO_FL_USER0
+// IMEMO/fields and T_OBJECT have exactly the same layout.
+// This is useful for JIT and common codepaths.
+#define OBJ_FIELD_HEAP ROBJECT_HEAP
+STATIC_ASSERT(imemo_fields_flags, OBJ_FIELD_HEAP == IMEMO_FL_USER0);
+STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.ary) == offsetof(struct rb_fields, as.embed.fields));
+STATIC_ASSERT(imemo_fields_external_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr));
+STATIC_ASSERT(imemo_fields_complex_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table));
+
 #define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields)
 
-VALUE rb_imemo_fields_new(VALUE owner, size_t capa);
-VALUE rb_imemo_fields_new_complex(VALUE owner, size_t capa);
-VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl);
+#define IMEMO_SUBCLASSES_HEAP IMEMO_FL_USER0
+
+struct rb_subclasses {
+    VALUE flags;
+    uint32_t count;
+    uint32_t capacity;
+    union {
+        VALUE *external;
+        VALUE embed[1];
+    } as;
+};
+
+static inline VALUE *
+rb_imemo_subclasses_entries(VALUE v)
+{
+    struct rb_subclasses *s = (struct rb_subclasses *)v;
+    return FL_TEST_RAW(v, IMEMO_SUBCLASSES_HEAP) ? s->as.external : s->as.embed;
+}
+
+VALUE rb_imemo_fields_new(VALUE owner, /* shape_id_t */ uint32_t shape_id, bool shareable);
+VALUE rb_imemo_subclasses_new(uint32_t capacity);
+VALUE rb_imemo_fields_new_complex(VALUE owner, /* shape_id_t */ uint32_t shape_id, size_t capa, bool shareable);
+VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, /* shape_id_t */ uint32_t shape_id, st_table *tbl, bool shareable);
 VALUE rb_imemo_fields_clone(VALUE fields_obj);
 void rb_imemo_fields_clear(VALUE fields_obj);
 
 static inline VALUE
 rb_imemo_fields_owner(VALUE fields_obj)
 {
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields));
+
     return CLASS_OF(fields_obj);
 }
 
 static inline VALUE *
-rb_imemo_fields_ptr(VALUE obj_fields)
+rb_imemo_fields_ptr(VALUE fields_obj)
 {
-    if (!obj_fields) {
+    if (!fields_obj) {
         return NULL;
     }
 
-    RUBY_ASSERT(IMEMO_TYPE_P(obj_fields, imemo_fields));
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT));
 
-    if (RB_UNLIKELY(FL_TEST_RAW(obj_fields, OBJ_FIELD_EXTERNAL))) {
-        return IMEMO_OBJ_FIELDS(obj_fields)->as.external.ptr;
+    if (UNLIKELY(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP))) {
+        return IMEMO_OBJ_FIELDS(fields_obj)->as.external.ptr;
     }
     else {
-        return IMEMO_OBJ_FIELDS(obj_fields)->as.embed.fields;
+        return IMEMO_OBJ_FIELDS(fields_obj)->as.embed.fields;
     }
 }
 
 static inline st_table *
-rb_imemo_fields_complex_tbl(VALUE obj_fields)
+rb_imemo_fields_complex_tbl(VALUE fields_obj)
 {
-    if (!obj_fields) {
+    if (!fields_obj) {
         return NULL;
     }
 
-    RUBY_ASSERT(IMEMO_TYPE_P(obj_fields, imemo_fields));
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT));
+    RUBY_ASSERT(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP));
+
+    // Some codepaths unconditionally access the fields_ptr, and assume it can be used as st_table if the
+    // shape is complex.
+    RUBY_ASSERT((st_table *)rb_imemo_fields_ptr(fields_obj) == IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table);
 
-    return IMEMO_OBJ_FIELDS(obj_fields)->as.complex.table;
+    return IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table;
 }
 
 #endif /* INTERNAL_IMEMO_H */