1 files changed, 89 insertions, 45 deletions

diff --git a/thread_pthread.h b/thread_pthread.h
index 0f8e4d6096..992e9fb080 100644
--- a/thread_pthread.h
+++ b/thread_pthread.h
@@ -17,16 +17,70 @@
 #define RB_NATIVETHREAD_LOCK_INIT PTHREAD_MUTEX_INITIALIZER
 #define RB_NATIVETHREAD_COND_INIT PTHREAD_COND_INITIALIZER
 
-// per-Thead scheduler helper data
+// TLS can not be accessed across .so on arm64 and perhaps ppc64le too.
+#if defined(__arm64__) || defined(__aarch64__) || defined(__powerpc64__)
+# define RB_THREAD_CURRENT_EC_NOINLINE
+#endif
+
+// this data should be protected by timer_th.waiting_lock
+struct rb_thread_sched_waiting {
+    enum thread_sched_waiting_flag {
+        thread_sched_waiting_none     = 0x00,
+        thread_sched_waiting_timeout  = 0x01,
+        thread_sched_waiting_io_read  = 0x02,
+        thread_sched_waiting_io_write = 0x08,
+        thread_sched_waiting_io_force = 0x40, // ignore readable
+    } flags;
+
+    struct {
+        // should be compat with hrtime.h
+#ifdef MY_RUBY_BUILD_MAY_TIME_TRAVEL
+        int128_t timeout;
+#else
+        uint64_t timeout;
+#endif
+        uint32_t event_serial;
+        int fd; // -1 for timeout only
+        int result;
+    } data;
+
+    // connected to timer_th.waiting
+    struct ccan_list_node node;
+};
+
+// per-Thread scheduler helper data
 struct rb_thread_sched_item {
-    union {
+    struct {
         struct ccan_list_node ubf;
-        struct ccan_list_node readyq; // protected by sched->lock
+
+        // connected to ractor->threads.sched.reqdyq
+        // locked by ractor->threads.sched.lock
+        struct ccan_list_node readyq;
+
+        // connected to vm->ractor.sched.timeslice_threads
+        // locked by vm->ractor.sched.lock
+        struct ccan_list_node timeslice_threads;
+
+        // connected to vm->ractor.sched.running_threads
+        // locked by vm->ractor.sched.lock
+        struct ccan_list_node running_threads;
+
+        // connected to vm->ractor.sched.zombie_threads
+        struct ccan_list_node zombie_threads;
     } node;
+
+    struct rb_thread_sched_waiting waiting_reason;
+    uint32_t event_serial;
+
+    bool finished;
+    bool malloc_stack;
+    void *context_stack;
+    struct coroutine_context *context;
 };
 
 struct rb_native_thread {
-    int id;
+    rb_atomic_t serial;
+    struct rb_vm_struct *vm;
 
     rb_nativethread_id_t thread_id;
 
@@ -54,6 +108,11 @@ struct rb_native_thread {
 #ifdef USE_SIGALTSTACK
     void *altstack;
 #endif
+
+    struct coroutine_context *nt_context;
+    int dedicated;
+
+    size_t machine_stack_maxsize;
 };
 
 #undef except
@@ -63,51 +122,33 @@ struct rb_native_thread {
 
 // per-Ractor
 struct rb_thread_sched {
-    /* fast path */
-
-    const struct rb_thread_struct *running; // running thread or NULL
-    rb_nativethread_lock_t lock;
+    rb_nativethread_lock_t lock_;
+#if VM_CHECK_MODE
+    struct rb_thread_struct *lock_owner;
+#endif
+    struct rb_thread_struct *running; // running thread or NULL
+    bool is_running;
+    bool is_running_timeslice;
+    bool enable_mn_threads;
 
-    /*
-     * slow path, protected by ractor->thread_sched->lock
-     * - @readyq - FIFO queue of threads waiting for running
-     * - @timer - it handles timeslices for @current.  It is any one thread
-     *   in @waitq, there is no @timer if @waitq is empty, but always
-     *   a @timer if @waitq has entries
-     * - @timer_err tracks timeslice limit, the timeslice only resets
-     *   when pthread_cond_timedwait returns ETIMEDOUT, so frequent
-     *   switching between contended/uncontended GVL won't reset the
-     *   timer.
-     */
     struct ccan_list_head readyq;
-    const struct rb_thread_struct *timer;
-    int timer_err;
-
-    /* yield */
-    rb_nativethread_cond_t switch_cond;
-    rb_nativethread_cond_t switch_wait_cond;
-    int need_yield;
-    int wait_yield;
+    int readyq_cnt;
+    // ractor scheduling
+    struct ccan_list_node grq_node;
 };
 
-#ifndef RB_THREAD_LOCAL_SPECIFIER_IS_UNSUPPORTED
-# if __STDC_VERSION__ >= 201112
-#   define RB_THREAD_LOCAL_SPECIFIER _Thread_local
-# elif defined(__GNUC__)
-  /* note that ICC (linux) and Clang are covered by __GNUC__ */
-#   define RB_THREAD_LOCAL_SPECIFIER __thread
-# endif
-#endif
-
-RUBY_SYMBOL_EXPORT_BEGIN
 #ifdef RB_THREAD_LOCAL_SPECIFIER
-# ifdef __APPLE__
-// on Darwin, TLS can not be accessed across .so
-struct rb_execution_context_struct *rb_current_ec(void);
-void rb_current_ec_set(struct rb_execution_context_struct *);
-# else
-RUBY_EXTERN RB_THREAD_LOCAL_SPECIFIER struct rb_execution_context_struct *ruby_current_ec;
-# endif
+  NOINLINE(void rb_current_ec_set(struct rb_execution_context_struct *));
+
+  # ifdef RB_THREAD_CURRENT_EC_NOINLINE
+    NOINLINE(struct rb_execution_context_struct *rb_current_ec(void));
+  # else
+    RUBY_EXTERN RB_THREAD_LOCAL_SPECIFIER struct rb_execution_context_struct *ruby_current_ec;
+
+    // for RUBY_DEBUG_LOG()
+    RUBY_EXTERN RB_THREAD_LOCAL_SPECIFIER rb_atomic_t ruby_nt_serial;
+    #define RUBY_NT_SERIAL 1
+  # endif
 #else
 typedef pthread_key_t native_tls_key_t;
 
@@ -128,6 +169,9 @@ native_tls_set(native_tls_key_t key, void *ptr)
 
 RUBY_EXTERN native_tls_key_t ruby_current_ec_key;
 #endif
-RUBY_SYMBOL_EXPORT_END
+
+struct rb_ractor_struct;
+void rb_ractor_sched_wait(struct rb_execution_context_struct *ec, struct rb_ractor_struct *cr, rb_unblock_function_t *ubf, void *ptr);
+void rb_ractor_sched_wakeup(struct rb_ractor_struct *r, struct rb_thread_struct *th);
 
 #endif /* RUBY_THREAD_PTHREAD_H */