1 files changed, 132 insertions, 53 deletions

diff --git a/internal/imemo.h b/internal/imemo.h
index a28ad92f7b..e8a5f0fc8e 100644
--- a/internal/imemo.h
+++ b/internal/imemo.h
@@ -10,6 +10,7 @@
  */
 #include "ruby/internal/config.h"
 #include <stddef.h>             /* for size_t */
+#include "id_table.h"
 #include "internal/array.h"     /* for rb_ary_hidden_new_fill */
 #include "ruby/internal/stdbool.h"     /* for bool */
 #include "ruby/ruby.h"          /* for rb_block_call_func_t */
@@ -24,6 +25,7 @@
 #define IMEMO_FL_USER3 FL_USER7
 #define IMEMO_FL_USER4 FL_USER8
 #define IMEMO_FL_USER5 FL_USER9
+#define IMEMO_FL_USER6 FL_USER10
 
 enum imemo_type {
     imemo_env            =  0,
@@ -35,11 +37,13 @@ enum imemo_type {
     imemo_ment           =  6,
     imemo_iseq           =  7,
     imemo_tmpbuf         =  8,
-    imemo_ast            =  9, // Obsolete due to the universal parser
-    imemo_parser_strterm = 10,
-    imemo_callinfo       = 11,
-    imemo_callcache      = 12,
-    imemo_constcache     = 13,
+    imemo_cvar_entry     =  9,
+    imemo_callinfo       = 10,
+    imemo_callcache      = 11,
+    imemo_constcache     = 12,
+    imemo_fields         = 13,
+    imemo_subclasses     = 14,
+    imemo_cdhash         = 15,
 };
 
 /* CREF (Class REFerence) is defined in method.h */
@@ -56,7 +60,6 @@ struct vm_svar {
 /*! THROW_DATA */
 struct vm_throw_data {
     VALUE flags;
-    VALUE reserved;
     const VALUE throw_obj;
     const struct rb_control_frame_struct *catch_frame;
     int throw_state;
@@ -92,30 +95,36 @@ struct vm_ifunc {
 
 struct rb_imemo_tmpbuf_struct {
     VALUE flags;
-    VALUE reserved;
     VALUE *ptr; /* malloc'ed buffer */
-    struct rb_imemo_tmpbuf_struct *next; /* next imemo */
-    size_t cnt; /* buffer size in VALUE */
+    size_t size; /* buffer size in bytes */
 };
 
+struct rb_imemo_cdhash {
+    VALUE flags;
+    st_table tbl;
+};
+
+/* Set on imemo_memo when u3 holds a VALUE that GC must mark.
+ * When unset, u3 is a non-VALUE (cnt/state). */
+#define MEMO_U3_IS_VALUE IMEMO_FL_USER0
+
 /*! MEMO
  *
  * @see imemo_type
  * */
 struct MEMO {
     VALUE flags;
-    VALUE reserved;
     const VALUE v1;
     const VALUE v2;
     union {
         long cnt;
         long state;
         const VALUE value;
-        void (*func)(void);
     } u3;
 };
 
-#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T)))
+#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), false))
+#define SHAREABLE_IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), true))
 
 /* ment is in method.h */
 
@@ -132,41 +141,27 @@ struct MEMO {
 #ifndef RUBY_RUBYPARSER_H
 typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t;
 #endif
-rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
+VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size, bool is_shareable);
+VALUE rb_imemo_tmpbuf_new(void);
+struct MEMO *rb_imemo_memo_new(VALUE a, VALUE b, long c);
+struct MEMO *rb_imemo_memo_new_value(VALUE a, VALUE b, VALUE c);
 struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc);
 static inline enum imemo_type imemo_type(VALUE imemo);
 static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type);
 static inline bool imemo_throw_data_p(VALUE imemo);
 static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data);
-static inline VALUE rb_imemo_tmpbuf_auto_free_pointer(void);
 static inline void *RB_IMEMO_TMPBUF_PTR(VALUE v);
-static inline void *rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr);
-static inline VALUE rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str);
 static inline void MEMO_V1_SET(struct MEMO *m, VALUE v);
 static inline void MEMO_V2_SET(struct MEMO *m, VALUE v);
 
 size_t rb_imemo_memsize(VALUE obj);
-void rb_cc_table_mark(VALUE klass);
 void rb_imemo_mark_and_move(VALUE obj, bool reference_updating);
-void rb_cc_table_free(VALUE klass);
 void rb_imemo_free(VALUE obj);
 
 RUBY_SYMBOL_EXPORT_BEGIN
-VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size);
 const char *rb_imemo_name(enum imemo_type type);
 RUBY_SYMBOL_EXPORT_END
 
-static inline struct MEMO *
-MEMO_NEW(VALUE a, VALUE b, VALUE c)
-{
-    struct MEMO *memo = IMEMO_NEW(struct MEMO, imemo_memo, 0);
-    *((VALUE *)&memo->v1) = a;
-    *((VALUE *)&memo->v2) = b;
-    *((VALUE *)&memo->u3.value) = c;
-
-    return memo;
-}
-
 static inline enum imemo_type
 imemo_type(VALUE imemo)
 {
@@ -202,12 +197,6 @@ rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data)
     return rb_vm_ifunc_new(func, data, 0, UNLIMITED_ARGUMENTS);
 }
 
-static inline VALUE
-rb_imemo_tmpbuf_auto_free_pointer(void)
-{
-    return rb_imemo_new(imemo_tmpbuf, 0, sizeof(rb_imemo_tmpbuf_t));
-}
-
 static inline void *
 RB_IMEMO_TMPBUF_PTR(VALUE v)
 {
@@ -215,30 +204,16 @@ RB_IMEMO_TMPBUF_PTR(VALUE v)
     return p->ptr;
 }
 
-static inline void *
-rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr)
-{
-    return ((rb_imemo_tmpbuf_t *)v)->ptr = ptr;
-}
-
 static inline VALUE
-rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str)
+rb_imemo_tmpbuf_new_from_an_RString(VALUE str)
 {
-    const void *src;
     VALUE imemo;
-    rb_imemo_tmpbuf_t *tmpbuf;
-    void *dst;
     size_t len;
 
     StringValue(str);
-    /* create tmpbuf to keep the pointer before xmalloc */
-    imemo = rb_imemo_tmpbuf_auto_free_pointer();
-    tmpbuf = (rb_imemo_tmpbuf_t *)imemo;
     len = RSTRING_LEN(str);
-    src = RSTRING_PTR(str);
-    dst = ruby_xmalloc(len);
-    memcpy(dst, src, len);
-    tmpbuf->ptr = dst;
+    rb_alloc_tmp_buffer(&imemo, len);
+    memcpy(RB_IMEMO_TMPBUF_PTR(imemo), RSTRING_PTR(str), len);
     return imemo;
 }
 
@@ -254,4 +229,108 @@ MEMO_V2_SET(struct MEMO *m, VALUE v)
     RB_OBJ_WRITE(m, &m->v2, v);
 }
 
+VALUE rb_imemo_cdhash_new(size_t size, const struct st_hash_type *type);
+
+static inline st_table *
+rb_imemo_cdhash_tbl(VALUE obj)
+{
+    RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_cdhash));
+    return &((struct rb_imemo_cdhash *)obj)->tbl;
+}
+
+struct rb_fields {
+    struct RBasic basic;
+    union {
+        struct {
+            VALUE fields[1];
+        } embed;
+        struct {
+            VALUE *ptr;
+        } external;
+        struct {
+            // Note: the st_table could be embedded, but complex T_CLASS should be rare to
+            // non-existent, so not really worth the trouble.
+            st_table *table;
+        } complex;
+    } as;
+};
+
+// IMEMO/fields and T_OBJECT have exactly the same layout.
+// This is useful for JIT and common codepaths.
+#define OBJ_FIELD_HEAP ROBJECT_HEAP
+STATIC_ASSERT(imemo_fields_flags, OBJ_FIELD_HEAP == IMEMO_FL_USER0);
+STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.ary) == offsetof(struct rb_fields, as.embed.fields));
+STATIC_ASSERT(imemo_fields_external_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr));
+STATIC_ASSERT(imemo_fields_complex_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table));
+
+#define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields)
+
+#define IMEMO_SUBCLASSES_HEAP IMEMO_FL_USER0
+
+struct rb_subclasses {
+    VALUE flags;
+    uint32_t count;
+    uint32_t capacity;
+    union {
+        VALUE *external;
+        VALUE embed[1];
+    } as;
+};
+
+static inline VALUE *
+rb_imemo_subclasses_entries(VALUE v)
+{
+    struct rb_subclasses *s = (struct rb_subclasses *)v;
+    return FL_TEST_RAW(v, IMEMO_SUBCLASSES_HEAP) ? s->as.external : s->as.embed;
+}
+
+VALUE rb_imemo_fields_new(VALUE owner, /* shape_id_t */ uint32_t shape_id, bool shareable);
+VALUE rb_imemo_subclasses_new(uint32_t capacity);
+VALUE rb_imemo_fields_new_complex(VALUE owner, /* shape_id_t */ uint32_t shape_id, size_t capa, bool shareable);
+VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, /* shape_id_t */ uint32_t shape_id, st_table *tbl, bool shareable);
+VALUE rb_imemo_fields_clone(VALUE fields_obj);
+void rb_imemo_fields_clear(VALUE fields_obj);
+
+static inline VALUE
+rb_imemo_fields_owner(VALUE fields_obj)
+{
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields));
+
+    return CLASS_OF(fields_obj);
+}
+
+static inline VALUE *
+rb_imemo_fields_ptr(VALUE fields_obj)
+{
+    if (!fields_obj) {
+        return NULL;
+    }
+
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT));
+
+    if (UNLIKELY(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP))) {
+        return IMEMO_OBJ_FIELDS(fields_obj)->as.external.ptr;
+    }
+    else {
+        return IMEMO_OBJ_FIELDS(fields_obj)->as.embed.fields;
+    }
+}
+
+static inline st_table *
+rb_imemo_fields_complex_tbl(VALUE fields_obj)
+{
+    if (!fields_obj) {
+        return NULL;
+    }
+
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT));
+    RUBY_ASSERT(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP));
+
+    // Some codepaths unconditionally access the fields_ptr, and assume it can be used as st_table if the
+    // shape is complex.
+    RUBY_ASSERT((st_table *)rb_imemo_fields_ptr(fields_obj) == IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table);
+
+    return IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table;
+}
+
 #endif /* INTERNAL_IMEMO_H */