ractor local storage C-API

To manage ractor-local data for C extension, the following APIs
are defined.

* rb_ractor_local_storage_value_newkey
* rb_ractor_local_storage_value
* rb_ractor_local_storage_value_set
* rb_ractor_local_storage_ptr_newkey
* rb_ractor_local_storage_ptr
* rb_ractor_local_storage_ptr_set

At first, you need to create a key of storage by
rb_ractor_local_(value|ptr)_newkey().
For ptr storage, it accepts the type of storage,
how to mark and how to free with ractor's lifetime.

rb_ractor_local_storage_value/set are used to access a VALUE
and rb_ractor_local_storage_ptr/set are used to access a pointer.

random.c uses this API.

6 files changed, 254 insertions, 36 deletions

diff --git a/common.mk b/common.mk
index 14e5ca8dce..6b11bec33b 100644
--- a/common.mk
+++ b/common.mk
@@ -10572,6 +10572,7 @@ random.$(OBJEXT): {$(VPATH)}missing.h
 random.$(OBJEXT): {$(VPATH)}mt19937.c
 random.$(OBJEXT): {$(VPATH)}onigmo.h
 random.$(OBJEXT): {$(VPATH)}oniguruma.h
+random.$(OBJEXT): {$(VPATH)}ractor.h
 random.$(OBJEXT): {$(VPATH)}random.c
 random.$(OBJEXT): {$(VPATH)}random.h
 random.$(OBJEXT): {$(VPATH)}ruby_atomic.h
diff --git a/gc.c b/gc.c
index ab3f8610ce..aae05b5795 100644
--- a/gc.c
+++ b/gc.c
@@ -7273,6 +7273,7 @@ gc_marks_finish(rb_objspace_t *objspace)
     }
 
     rb_transient_heap_finish_marking();
+    rb_ractor_finish_marking();
 
     gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0);
 
diff --git a/include/ruby/ractor.h b/include/ruby/ractor.h
index 3b4552edef..327d4f4126 100644
--- a/include/ruby/ractor.h
+++ b/include/ruby/ractor.h
@@ -12,6 +12,14 @@
  *             file COPYING are met.  Consult the file for details.
  */
 
+struct rb_ractor_local_storage_type {
+    void (*mark)(void *ptr);
+    void (*free)(void *ptr);
+    // TODO: update
+};
+
+typedef struct rb_ractor_local_key_struct *rb_ractor_local_key_t;
+
 RUBY_SYMBOL_EXPORT_BEGIN
 RUBY_EXTERN VALUE rb_cRactor;
 
@@ -22,6 +30,17 @@ void rb_ractor_stdin_set(VALUE);
 void rb_ractor_stdout_set(VALUE);
 void rb_ractor_stderr_set(VALUE);
 
+rb_ractor_local_key_t rb_ractor_local_storage_value_newkey(void);
+VALUE rb_ractor_local_storage_value(rb_ractor_local_key_t key);
+void  rb_ractor_local_storage_value_set(rb_ractor_local_key_t key, VALUE val);
+
+RUBY_EXTERN const struct rb_ractor_local_storage_type rb_ractor_local_storage_type_free;
+#define RB_RACTOR_LOCAL_STORAGE_TYPE_FREE (&rb_ractor_local_storage_type_free)
+
+rb_ractor_local_key_t rb_ractor_local_storage_ptr_newkey(const struct rb_ractor_local_storage_type *type);
+void *rb_ractor_local_storage_ptr(rb_ractor_local_key_t key);
+void  rb_ractor_local_storage_ptr_set(rb_ractor_local_key_t key, void *ptr);
+
 RUBY_SYMBOL_EXPORT_END
 
 #define RB_OBJ_SHAREABLE_P(obj) FL_TEST_RAW((obj), RUBY_FL_SHAREABLE)
diff --git a/ractor.c b/ractor.c
index 6b46d4ba95..4a75b41fc5 100644
--- a/ractor.c
+++ b/ractor.c
@@ -177,6 +177,9 @@ ractor_queue_mark(struct rb_ractor_queue *rq)
     }
 }
 
+static void ractor_local_storage_mark(rb_ractor_t *r);
+static void ractor_local_storage_free(rb_ractor_t *r);
+
 static void
 ractor_mark(void *ptr)
 {
@@ -201,10 +204,7 @@ ractor_mark(void *ptr)
         }
     }
 
-    if (r->default_rand) {
-        void rb_default_rand_mark(void *); // random.c
-        rb_default_rand_mark(r->default_rand);
-    }
+    ractor_local_storage_mark(r);
 }
 
 static void
@@ -227,7 +227,7 @@ ractor_free(void *ptr)
     rb_native_cond_destroy(&r->wait.cond);
     ractor_queue_free(&r->incoming_queue);
     ractor_waiting_list_free(&r->taking_ractors);
-    if (r->default_rand) ruby_xfree(r->default_rand);
+    ractor_local_storage_free(r);
     ruby_xfree(r);
 }
 
@@ -1773,24 +1773,6 @@ rb_ractor_stderr_set(VALUE err)
     }
 }
 
-struct rb_random_struct *
-rb_ractor_default_rand(struct rb_random_struct *ptr)
-{
-    if (rb_ractor_main_p()) {
-        static struct rb_random_struct *default_rnd;
-        if (UNLIKELY(ptr != NULL)) {
-            rb_ractor_t *cr = GET_RACTOR();
-            cr->default_rand = default_rnd = ptr;
-        }
-        return default_rnd;
-    }
-    else {
-        rb_ractor_t *cr = GET_RACTOR();
-        if (UNLIKELY(ptr != NULL)) cr->default_rand = ptr;
-        return cr->default_rand;
-    }
-}
-
 /// traverse function
 
 // 2: stop search
@@ -2566,4 +2548,200 @@ static VALUE ractor_copy(VALUE obj)
     }
 }
 
+// Ractor local storage
+
+struct rb_ractor_local_key_struct {
+    const struct rb_ractor_local_storage_type *type;
+    void *main_cache;
+};
+
+static struct freed_ractor_local_keys_struct {
+    int cnt;
+    int capa;
+    rb_ractor_local_key_t *keys;
+} freed_ractor_local_keys;
+
+static int
+ractor_local_storage_mark_i(st_data_t key, st_data_t val, st_data_t dmy)
+{
+    struct rb_ractor_local_key_struct *k = (struct rb_ractor_local_key_struct *)key;
+    if (k->type->mark) (*k->type->mark)((void *)val);
+    return ST_CONTINUE;
+}
+
+static void
+ractor_local_storage_mark(rb_ractor_t *r)
+{
+    if (r->local_storage) {
+        st_foreach(r->local_storage, ractor_local_storage_mark_i, 0);
+
+        for (int i=0; i<freed_ractor_local_keys.cnt; i++) {
+            rb_ractor_local_key_t key = freed_ractor_local_keys.keys[i];
+            st_data_t val;
+            if (st_delete(r->local_storage, (st_data_t *)&key, &val) &&
+                key->type->free) {
+                (*key->type->free)((void *)val);
+            }
+        }
+    }
+}
+
+static int
+ractor_local_storage_free_i(st_data_t key, st_data_t val, st_data_t dmy)
+{
+    struct rb_ractor_local_key_struct *k = (struct rb_ractor_local_key_struct *)key;
+    if (k->type->free) (*k->type->free)((void *)val);
+    return ST_CONTINUE;
+}
+
+static void
+ractor_local_storage_free(rb_ractor_t *r)
+{
+    if (r->local_storage) {
+        st_foreach(r->local_storage, ractor_local_storage_free_i, 0);
+        st_free_table(r->local_storage);
+    }
+}
+
+static void
+rb_ractor_local_storage_value_mark(void *ptr)
+{
+    rb_gc_mark((VALUE)ptr);
+}
+
+static const struct rb_ractor_local_storage_type ractor_local_storage_type_null = {
+    NULL,
+    NULL,
+};
+
+const struct rb_ractor_local_storage_type rb_ractor_local_storage_type_free = {
+    NULL,
+    ruby_xfree,
+};
+
+static const struct rb_ractor_local_storage_type ractor_local_storage_type_value = {
+    rb_ractor_local_storage_value_mark,
+    NULL,
+};
+
+rb_ractor_local_key_t
+rb_ractor_local_storage_ptr_newkey(const struct rb_ractor_local_storage_type *type)
+{
+    rb_ractor_local_key_t key = ALLOC(struct rb_ractor_local_key_struct);
+    key->type = type ? type : &ractor_local_storage_type_null;
+    key->main_cache = (void *)Qundef;
+    return key;
+}
+
+rb_ractor_local_key_t
+rb_ractor_local_storage_value_newkey(void)
+{
+    return rb_ractor_local_storage_ptr_newkey(&ractor_local_storage_type_value);
+}
+
+void
+rb_ractor_local_storage_delkey(rb_ractor_local_key_t key)
+{
+    RB_VM_LOCK_ENTER();
+    {
+        if (freed_ractor_local_keys.cnt == freed_ractor_local_keys.capa) {
+            freed_ractor_local_keys.capa = freed_ractor_local_keys.capa ? freed_ractor_local_keys.capa * 2 : 4;
+            REALLOC_N(freed_ractor_local_keys.keys, rb_ractor_local_key_t, freed_ractor_local_keys.capa);
+        }
+        freed_ractor_local_keys.keys[freed_ractor_local_keys.cnt++] = key;
+    }
+    RB_VM_LOCK_LEAVE();
+}
+
+static bool
+ractor_local_ref(rb_ractor_local_key_t key, void **pret)
+{
+    if (rb_ractor_main_p()) {
+        if ((VALUE)key->main_cache != Qundef) {
+            *pret = key->main_cache;
+            return true;
+        }
+        else {
+            return false;
+        }
+    }
+    else {
+        rb_ractor_t *cr = GET_RACTOR();
+
+        if (cr->local_storage && st_lookup(cr->local_storage, (st_data_t)key, (st_data_t *)pret)) {
+            return true;
+        }
+        else {
+            return false;
+        }
+    }
+}
+
+static void
+ractor_local_set(rb_ractor_local_key_t key, void *ptr)
+{
+    rb_ractor_t *cr = GET_RACTOR();
+
+    if (cr->local_storage == NULL) {
+        cr->local_storage = st_init_numtable();
+    }
+
+    st_insert(cr->local_storage, (st_data_t)key, (st_data_t)ptr);
+
+    if (rb_ractor_main_p()) {
+        key->main_cache = ptr;
+    }
+}
+
+VALUE
+rb_ractor_local_storage_value(rb_ractor_local_key_t key)
+{
+    VALUE val;
+    if (ractor_local_ref(key, (void **)&val)) {
+        return val;
+    }
+    else {
+        return Qnil;
+    }
+}
+
+void
+rb_ractor_local_storage_value_set(rb_ractor_local_key_t key, VALUE val)
+{
+    ractor_local_set(key, (void *)val);
+}
+
+void *
+rb_ractor_local_storage_ptr(rb_ractor_local_key_t key)
+{
+    void *ret;
+    if (ractor_local_ref(key, &ret)) {
+        return ret;
+    }
+    else {
+        return NULL;
+    }
+}
+
+void
+rb_ractor_local_storage_ptr_set(rb_ractor_local_key_t key, void *ptr)
+{
+    ractor_local_set(key, ptr);
+}
+
+#define DEFAULT_KEYS_CAPA 0x10
+
+void
+rb_ractor_finish_marking(void)
+{
+    for (int i=0; i<freed_ractor_local_keys.cnt; i++) {
+        ruby_xfree(freed_ractor_local_keys.keys[i]);
+    }
+    freed_ractor_local_keys.cnt = 0;
+    if (freed_ractor_local_keys.capa > DEFAULT_KEYS_CAPA) {
+        freed_ractor_local_keys.capa = DEFAULT_KEYS_CAPA;
+        REALLOC_N(freed_ractor_local_keys.keys, rb_ractor_local_key_t, DEFAULT_KEYS_CAPA);
+    }
+}
+
 #include "ractor.rbinc"
diff --git a/ractor_core.h b/ractor_core.h
index 6e9a63dc28..b0295155ac 100644
--- a/ractor_core.h
+++ b/ractor_core.h
@@ -123,14 +123,16 @@ struct rb_ractor_struct {
 
     struct list_node vmlr_node;
 
+    // ractor local data
+
+    st_table *local_storage;
+
     VALUE r_stdin;
     VALUE r_stdout;
     VALUE r_stderr;
     VALUE verbose;
     VALUE debug;
 
-    struct rb_random_struct *default_rand; // used in random.c
-
     // gc.c rb_objspace_reachable_objects_from
     struct gc_mark_func_data_struct {
         void *data;
@@ -163,9 +165,15 @@ void rb_ractor_vm_barrier_interrupt_running_thread(rb_ractor_t *r);
 void rb_ractor_terminate_interrupt_main_thread(rb_ractor_t *r);
 void rb_ractor_terminate_all(void);
 bool rb_ractor_main_p_(void);
+void rb_ractor_finish_marking(void);
 
 RUBY_SYMBOL_EXPORT_BEGIN
 bool rb_ractor_shareable_p_continue(VALUE obj);
+
+// THIS FUNCTION SHOULD NOT CALL WHILE INCREMENTAL MARKING!!
+// This function is for T_DATA::free_func
+void rb_ractor_local_storage_delkey(rb_ractor_local_key_t key);
+
 RUBY_SYMBOL_EXPORT_END
 
 RUBY_EXTERN bool ruby_multi_ractor;
diff --git a/random.c b/random.c
index 6b26e2b6cd..79210fe3cb 100644
--- a/random.c
+++ b/random.c
@@ -63,6 +63,7 @@
 #include "internal/sanitizers.h"
 #include "ruby_atomic.h"
 #include "ruby/random.h"
+#include "ruby/ractor.h"
 
 typedef int int_must_be_32bit_at_least[sizeof(int) * CHAR_BIT < 32 ? -1 : 1];
 
@@ -146,23 +147,31 @@ rand_start(rb_random_mt_t *r)
     return &rand_mt_start(r)->base;
 }
 
+static rb_ractor_local_key_t default_rand_key;
+
+static void
+default_rand_mark(void *ptr)
+{
+    rb_random_mt_t *rnd = (rb_random_mt_t *)ptr;
+    rb_gc_mark(rnd->base.seed);
+}
+
+static const struct rb_ractor_local_storage_type default_rand_key_storage_type = {
+    default_rand_mark,
+    ruby_xfree,
+};
+
 static rb_random_mt_t *
 default_rand(void)
 {
-    rb_random_t *rb_ractor_default_rand(rb_random_t *); // ractor.c
-    rb_random_mt_t *rnd = (rb_random_mt_t *)rb_ractor_default_rand(NULL);
-    if (rnd == NULL) {
+    rb_random_mt_t *rnd;
+
+    if ((rnd = rb_ractor_local_storage_ptr(default_rand_key)) == NULL) {
         rnd = ZALLOC(rb_random_mt_t);
-        rb_ractor_default_rand(&rnd->base);
+        rb_ractor_local_storage_ptr_set(default_rand_key, rnd);
     }
-    return rnd;
-}
 
-void
-rb_default_rand_mark(void *ptr)
-{
-    rb_random_mt_t *rnd = (rb_random_mt_t *)ptr;
-    rb_gc_mark(rnd->base.seed);
+    return rnd;
 }
 
 static rb_random_mt_t *
@@ -1727,6 +1736,8 @@ InitVM_Random(void)
 	rb_define_method(m, "random_number", rand_random_number, -1);
 	rb_define_method(m, "rand", rand_random_number, -1);
     }
+
+    default_rand_key = rb_ractor_local_storage_ptr_newkey(&default_rand_key_storage_type);
 }
 
 #undef rb_intern