diff options
Diffstat (limited to 'cont.c')
| -rw-r--r-- | cont.c | 690 |
1 files changed, 448 insertions, 242 deletions
@@ -30,25 +30,31 @@ extern int madvise(caddr_t, size_t, int); #include "internal/cont.h" #include "internal/thread.h" #include "internal/error.h" +#include "internal/eval.h" #include "internal/gc.h" #include "internal/proc.h" #include "internal/sanitizers.h" #include "internal/warnings.h" #include "ruby/fiber/scheduler.h" -#include "rjit.h" #include "yjit.h" #include "vm_core.h" #include "vm_sync.h" #include "id_table.h" #include "ractor_core.h" +#include "zjit.h" -static const int DEBUG = 0; +enum { + DEBUG = 0, + DEBUG_EXPAND = 0, + DEBUG_ACQUIRE = 0, +}; #define RB_PAGE_SIZE (pagesize) #define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1)) static long pagesize; -static const rb_data_type_t cont_data_type, fiber_data_type; +static const rb_data_type_t rb_cont_data_type; +static const rb_data_type_t rb_fiber_data_type; static VALUE rb_cContinuation; static VALUE rb_cFiber; static VALUE rb_eFiberError; @@ -61,11 +67,11 @@ static VALUE rb_cFiberPool; // Defined in `coroutine/$arch/Context.h`: #ifdef COROUTINE_LIMITED_ADDRESS_SPACE #define FIBER_POOL_ALLOCATION_FREE -#define FIBER_POOL_INITIAL_SIZE 8 -#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 32 +#define FIBER_POOL_MINIMUM_COUNT 8 +#define FIBER_POOL_MAXIMUM_ALLOCATIONS 32 #else -#define FIBER_POOL_INITIAL_SIZE 32 -#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 1024 +#define FIBER_POOL_MINIMUM_COUNT 32 +#define FIBER_POOL_MAXIMUM_ALLOCATIONS 1024 #endif #ifdef RB_EXPERIMENTAL_FIBER_POOL #define FIBER_POOL_ALLOCATION_FREE @@ -78,6 +84,7 @@ enum context_type { struct cont_saved_vm_stack { VALUE *ptr; + size_t size; #ifdef CAPTURE_JUST_VALID_VM_STACK size_t slen; /* length of stack (head of ec->vm_stack) */ size_t clen; /* length of control frames (tail of ec->vm_stack) */ @@ -187,7 +194,11 @@ struct fiber_pool { size_t count; // The initial number of stacks to allocate. - size_t initial_count; + size_t minimum_count; + + // If positive, total stacks in this pool cannot exceed this (shared pool only: + // set via RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT). Expansion fails with errno EAGAIN. + size_t maximum_count; // Whether to madvise(free) the stack or not. // If this value is set to 1, the stack will be madvise(free)ed @@ -226,7 +237,6 @@ typedef struct rb_context_struct { } machine; rb_execution_context_t saved_ec; rb_jmpbuf_t jmpbuf; - rb_ensure_entry_t *ensure_array; struct rb_jit_cont *jit_cont; // Continuation contexts for JITs } rb_context_t; @@ -278,7 +288,12 @@ static struct fiber_pool shared_fiber_pool = {NULL, NULL, 0, 0, 0, 0}; void rb_free_shared_fiber_pool(void) { - xfree(shared_fiber_pool.allocations); + struct fiber_pool_allocation *allocations = shared_fiber_pool.allocations; + while (allocations) { + struct fiber_pool_allocation *next = allocations->next; + SIZED_FREE(allocations); + allocations = next; + } } static ID fiber_initialize_keywords[3] = {0}; @@ -464,11 +479,12 @@ fiber_pool_allocate_memory(size_t * count, size_t stride) // the system would allow (e.g. overcommit * physical memory + swap), we // divide count by two and try again. This condition should only be // encountered in edge cases, but we handle it here gracefully. - while (*count > 1) { + while (*count) { #if defined(_WIN32) void * base = VirtualAlloc(0, (*count)*stride, MEM_COMMIT, PAGE_READWRITE); if (!base) { + errno = rb_w32_map_errno(GetLastError()); *count = (*count) >> 1; } else { @@ -476,18 +492,20 @@ fiber_pool_allocate_memory(size_t * count, size_t stride) } #else errno = 0; - void * base = mmap(NULL, (*count)*stride, PROT_READ | PROT_WRITE, FIBER_STACK_FLAGS, -1, 0); + size_t mmap_size = (*count)*stride; + void * base = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, FIBER_STACK_FLAGS, -1, 0); if (base == MAP_FAILED) { // If the allocation fails, count = count / 2, and try again. *count = (*count) >> 1; } else { + ruby_annotate_mmap(base, mmap_size, "Ruby:fiber_pool_allocate_memory"); #if defined(MADV_FREE_REUSE) // On Mac MADV_FREE_REUSE is necessary for the task_info api // to keep the accounting accurate as possible when a page is marked as reusable // it can possibly not occurring at first call thus re-iterating if necessary. - while (madvise(base, (*count)*stride, MADV_FREE_REUSE) == -1 && errno == EAGAIN); + while (madvise(base, mmap_size, MADV_FREE_REUSE) == -1 && errno == EAGAIN); #endif return base; } @@ -498,26 +516,53 @@ fiber_pool_allocate_memory(size_t * count, size_t stride) } // Given an existing fiber pool, expand it by the specified number of stacks. +// // @param count the maximum number of stacks to allocate. -// @return the allocated fiber pool. +// @return the new allocation on success, or NULL on failure with errno set. +// @raise NoMemoryError if the struct or memory allocation fails. +// +// Call from fiber_pool_stack_acquire_expand with VM lock held, or from +// fiber_pool_initialize before the pool is shared across threads. // @sa fiber_pool_allocation_free static struct fiber_pool_allocation * fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count) { + if (count == 0) { + errno = EAGAIN; + return NULL; + } + STACK_GROW_DIR_DETECTION; size_t size = fiber_pool->size; size_t stride = size + RB_PAGE_SIZE; + // If the maximum number of stacks is set, and we have reached it, return NULL. + if (fiber_pool->maximum_count > 0) { + if (fiber_pool->count >= fiber_pool->maximum_count) { + errno = EAGAIN; + return NULL; + } + size_t remaining = fiber_pool->maximum_count - fiber_pool->count; + if (count > remaining) { + count = remaining; + } + } + + // Allocate metadata before mmap: ruby_xmalloc (RB_ALLOC) raises on failure and + // must not run after base is mapped, or the region would leak. + struct fiber_pool_allocation * allocation = RB_ALLOC(struct fiber_pool_allocation); + // Allocate the memory required for the stacks: void * base = fiber_pool_allocate_memory(&count, stride); if (base == NULL) { - rb_raise(rb_eFiberError, "can't alloc machine stack to fiber (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, size, ERRNOMSG); + if (!errno) errno = ENOMEM; + ruby_xfree(allocation); + return NULL; } struct fiber_pool_vacancy * vacancies = fiber_pool->vacancies; - struct fiber_pool_allocation * allocation = RB_ALLOC(struct fiber_pool_allocation); // Initialize fiber pool allocation: allocation->base = base; @@ -529,7 +574,7 @@ fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count) #endif allocation->pool = fiber_pool; - if (DEBUG) { + if (DEBUG_EXPAND) { fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n", count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size); } @@ -538,18 +583,27 @@ fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count) for (size_t i = 0; i < count; i += 1) { void * base = (char*)allocation->base + (stride * i); void * page = (char*)base + STACK_DIR_UPPER(size, 0); - #if defined(_WIN32) DWORD old_protect; if (!VirtualProtect(page, RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) { + int error = rb_w32_map_errno(GetLastError()); VirtualFree(allocation->base, 0, MEM_RELEASE); - rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG); + ruby_xfree(allocation); + errno = error; + return NULL; } +#elif defined(__wasi__) + // wasi-libc's mprotect emulation doesn't support PROT_NONE. + (void)page; #else if (mprotect(page, RB_PAGE_SIZE, PROT_NONE) < 0) { + int error = errno; + if (!error) error = ENOMEM; munmap(allocation->base, count*stride); - rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG); + ruby_xfree(allocation); + errno = error; + return NULL; } #endif @@ -585,7 +639,7 @@ fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count) // Initialize the specified fiber pool with the given number of stacks. // @param vm_stack_size The size of the vm stack to allocate. static void -fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, size_t vm_stack_size) +fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t minimum_count, size_t maximum_count, size_t vm_stack_size) { VM_ASSERT(vm_stack_size < size); @@ -593,13 +647,17 @@ fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, fiber_pool->vacancies = NULL; fiber_pool->size = ((size / RB_PAGE_SIZE) + 1) * RB_PAGE_SIZE; fiber_pool->count = 0; - fiber_pool->initial_count = count; + fiber_pool->minimum_count = minimum_count; + fiber_pool->maximum_count = maximum_count; fiber_pool->free_stacks = 1; fiber_pool->used = 0; - fiber_pool->vm_stack_size = vm_stack_size; - fiber_pool_expand(fiber_pool, count); + if (fiber_pool->minimum_count > 0) { + if (RB_UNLIKELY(!fiber_pool_expand(fiber_pool, fiber_pool->minimum_count))) { + rb_raise(rb_eFiberError, "can't allocate initial fiber stacks (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", fiber_pool->minimum_count, fiber_pool->size, strerror(errno)); + } + } } #ifdef FIBER_POOL_ALLOCATION_FREE @@ -643,49 +701,120 @@ fiber_pool_allocation_free(struct fiber_pool_allocation * allocation) allocation->pool->count -= allocation->count; - ruby_xfree(allocation); + SIZED_FREE(allocation); } #endif -// Acquire a stack from the given fiber pool. If none are available, allocate more. -static struct fiber_pool_stack -fiber_pool_stack_acquire(struct fiber_pool * fiber_pool) +// Number of stacks to request when expanding the pool (clamped to min/max). +static size_t +fiber_pool_stack_expand_count(const struct fiber_pool *pool) { - struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pop(fiber_pool); + const size_t maximum_allocations = FIBER_POOL_MAXIMUM_ALLOCATIONS; + const size_t minimum_count = FIBER_POOL_MINIMUM_COUNT; - if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: %p used=%"PRIuSIZE"\n", (void*)fiber_pool->vacancies, fiber_pool->used); + // We are going try and double the number of stacks in the pool: + size_t count = pool->count; + if (count > maximum_allocations) count = maximum_allocations; + if (count < minimum_count) count = minimum_count; - if (!vacancy) { - const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE; - const size_t minimum = fiber_pool->initial_count; + // If we have a maximum count, we need to clamp the number of stacks to the maximum: + if (pool->maximum_count > 0) { + if (pool->count >= pool->maximum_count) { + // No expansion is possible: + return 0; + } - size_t count = fiber_pool->count; - if (count > maximum) count = maximum; - if (count < minimum) count = minimum; + // Otherwise, compute the number of stacks we can allocate to bring us to the maximum: + size_t remaining = pool->maximum_count - pool->count; + if (count > remaining) { + count = remaining; + } + } - fiber_pool_expand(fiber_pool, count); + return count; +} - // The free list should now contain some stacks: - VM_ASSERT(fiber_pool->vacancies); +// When the vacancy list is empty, grow the pool (and run GC only if mmap fails). Caller holds the VM lock. +// Returns NULL if expansion failed after GC + retry; errno is set. Otherwise returns a vacancy. +static struct fiber_pool_vacancy * +fiber_pool_stack_acquire_expand(struct fiber_pool *fiber_pool) +{ + size_t count = fiber_pool_stack_expand_count(fiber_pool); + + if (DEBUG_ACQUIRE) fprintf(stderr, "fiber_pool_stack_acquire: expanding fiber pool by %"PRIuSIZE" stacks\n", count); + + struct fiber_pool_vacancy *vacancy = NULL; + + if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) { + return fiber_pool_vacancy_pop(fiber_pool); + } + else { + if (DEBUG_ACQUIRE) fprintf(stderr, "fiber_pool_stack_acquire: expand failed (%s), collecting garbage\n", strerror(errno)); + + rb_gc(); + // After running GC, the vacancy list may have some stacks: vacancy = fiber_pool_vacancy_pop(fiber_pool); + if (RB_LIKELY(vacancy)) { + return vacancy; + } + + // Recompute count as gc may have freed up some allocations: + count = fiber_pool_stack_expand_count(fiber_pool); + + // Try to expand the fiber pool again: + if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) { + return fiber_pool_vacancy_pop(fiber_pool); + } + else { + // Okay, we really failed to acquire a stack. Give up and return NULL with errno set: + return NULL; + } } +} + +// Acquire a stack from the given fiber pool. If none are available, allocate more. +static struct fiber_pool_stack +fiber_pool_stack_acquire(struct fiber_pool * fiber_pool) +{ + struct fiber_pool_vacancy * vacancy; + + unsigned int lev; + RB_VM_LOCK_ENTER_LEV(&lev); + { + // Fast path: try to acquire a stack from the vacancy list: + vacancy = fiber_pool_vacancy_pop(fiber_pool); + + if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: %p used=%"PRIuSIZE"\n", (void*)fiber_pool->vacancies, fiber_pool->used); + + // Slow path: If the pool has no vacancies, expand first. Only run GC when expansion fails (e.g. mmap), so we can reclaim stacks from dead fibers before retrying: + if (RB_UNLIKELY(!vacancy)) { + vacancy = fiber_pool_stack_acquire_expand(fiber_pool); + + // If expansion failed, raise an error: + if (RB_UNLIKELY(!vacancy)) { + RB_VM_LOCK_LEAVE_LEV(&lev); + rb_raise(rb_eFiberError, "can't allocate fiber stack: %s", strerror(errno)); + } + } - VM_ASSERT(vacancy); - VM_ASSERT(vacancy->stack.base); + VM_ASSERT(vacancy); + VM_ASSERT(vacancy->stack.base); #if defined(COROUTINE_SANITIZE_ADDRESS) - __asan_unpoison_memory_region(fiber_pool_stack_poison_base(&vacancy->stack), fiber_pool_stack_poison_size(&vacancy->stack)); + __asan_unpoison_memory_region(fiber_pool_stack_poison_base(&vacancy->stack), fiber_pool_stack_poison_size(&vacancy->stack)); #endif - // Take the top item from the free list: - fiber_pool->used += 1; + // Take the top item from the free list: + fiber_pool->used += 1; #ifdef FIBER_POOL_ALLOCATION_FREE - vacancy->stack.allocation->used += 1; + vacancy->stack.allocation->used += 1; #endif - fiber_pool_stack_reset(&vacancy->stack); + fiber_pool_stack_reset(&vacancy->stack); + } + RB_VM_LOCK_LEAVE_LEV(&lev); return vacancy->stack; } @@ -796,6 +925,9 @@ static inline void ec_switch(rb_thread_t *th, rb_fiber_t *fiber) { rb_execution_context_t *ec = &fiber->cont.saved_ec; +#ifdef RUBY_ASAN_ENABLED + ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle(); +#endif rb_ractor_set_current_ec(th->ractor, th->ec = ec); // ruby_current_execution_context_ptr = th->ec = ec; @@ -818,6 +950,10 @@ fiber_restore_thread(rb_thread_t *th, rb_fiber_t *fiber) VM_ASSERT(th->ec->fiber_ptr == fiber); } +#ifndef COROUTINE_DECL +# define COROUTINE_DECL COROUTINE +#endif +NORETURN(static COROUTINE_DECL fiber_entry(struct coroutine_context * from, struct coroutine_context * to)); static COROUTINE fiber_entry(struct coroutine_context * from, struct coroutine_context * to) { @@ -895,6 +1031,17 @@ fiber_stack_release(rb_fiber_t * fiber) rb_ec_clear_vm_stack(ec); } +static void +fiber_stack_release_locked(rb_fiber_t *fiber) +{ + if (!ruby_vm_during_cleanup) { + // We can't try to acquire the VM lock here because MMTK calls free in its own native thread which has no ec. + // This assertion will fail on MMTK but we currently don't have CI for debug releases of MMTK, so we can assert for now. + ASSERT_vm_locking_with_barrier(); + } + fiber_stack_release(fiber); +} + static const char * fiber_status_name(enum fiber_status s) { @@ -916,7 +1063,9 @@ fiber_verify(const rb_fiber_t *fiber) switch (fiber->status) { case FIBER_RESUMED: - VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL); + if (fiber->cont.saved_ec.thread_ptr->self == 0) { + VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL); + } break; case FIBER_SUSPENDED: VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL); @@ -946,7 +1095,7 @@ cont_ptr(VALUE obj) { rb_context_t *cont; - TypedData_Get_Struct(obj, rb_context_t, &cont_data_type, cont); + TypedData_Get_Struct(obj, rb_context_t, &rb_cont_data_type, cont); return cont; } @@ -956,7 +1105,7 @@ fiber_ptr(VALUE obj) { rb_fiber_t *fiber; - TypedData_Get_Struct(obj, rb_fiber_t, &fiber_data_type, fiber); + TypedData_Get_Struct(obj, rb_fiber_t, &rb_fiber_data_type, fiber); if (!fiber) rb_raise(rb_eFiberError, "uninitialized fiber"); return fiber; @@ -1023,13 +1172,8 @@ cont_mark(void *ptr) cont->machine.stack + cont->machine.stack_size); } else { - /* fiber */ - const rb_fiber_t *fiber = (rb_fiber_t*)cont; - - if (!FIBER_TERMINATED_P(fiber)) { - rb_gc_mark_locations(cont->machine.stack, - cont->machine.stack + cont->machine.stack_size); - } + /* fiber machine context is marked as part of rb_execution_context_mark, no need to + * do anything here. */ } } @@ -1054,22 +1198,26 @@ cont_free(void *ptr) RUBY_FREE_ENTER("cont"); if (cont->type == CONTINUATION_CONTEXT) { - ruby_xfree(cont->saved_ec.vm_stack); - ruby_xfree(cont->ensure_array); - RUBY_FREE_UNLESS_NULL(cont->machine.stack); + SIZED_FREE_N(cont->saved_ec.vm_stack, cont->saved_ec.vm_stack_size); + SIZED_FREE_N(cont->machine.stack, cont->machine.stack_size); } else { rb_fiber_t *fiber = (rb_fiber_t*)cont; coroutine_destroy(&fiber->context); - fiber_stack_release(fiber); + fiber_stack_release_locked(fiber); } - RUBY_FREE_UNLESS_NULL(cont->saved_vm_stack.ptr); + SIZED_FREE_N(cont->saved_vm_stack.ptr, cont->saved_vm_stack.size); VM_ASSERT(cont->jit_cont != NULL); jit_cont_free(cont->jit_cont); /* free rb_cont_t or rb_fiber_t */ - ruby_xfree(ptr); + if (cont->type == CONTINUATION_CONTEXT) { + SIZED_FREE(cont); + } + else { + SIZED_FREE((rb_fiber_t *)cont); + } RUBY_FREE_LEAVE("cont"); } @@ -1110,12 +1258,7 @@ rb_fiber_update_self(rb_fiber_t *fiber) void rb_fiber_mark_self(const rb_fiber_t *fiber) { - if (fiber->cont.self) { - rb_gc_mark_movable(fiber->cont.self); - } - else { - rb_execution_context_mark(&fiber->cont.saved_ec); - } + rb_gc_mark_movable(fiber->cont.self); } static void @@ -1181,12 +1324,13 @@ fiber_memsize(const void *ptr) VALUE rb_obj_is_fiber(VALUE obj) { - return RBOOL(rb_typeddata_is_kind_of(obj, &fiber_data_type)); + return RBOOL(rb_typeddata_is_kind_of(obj, &rb_fiber_data_type)); } static void cont_save_machine_stack(rb_thread_t *th, rb_context_t *cont) { + const size_t old_stack_size = cont->machine.stack_size; size_t size; SET_MACHINE_STACK_END(&th->ec->machine.stack_end); @@ -1201,10 +1345,10 @@ cont_save_machine_stack(rb_thread_t *th, rb_context_t *cont) } if (cont->machine.stack) { - REALLOC_N(cont->machine.stack, VALUE, size); + SIZED_REALLOC_N(cont->machine.stack, VALUE, cont->machine.stack_size, old_stack_size); } else { - cont->machine.stack = ALLOC_N(VALUE, size); + cont->machine.stack = ALLOC_N(VALUE, cont->machine.stack_size); } FLUSH_REGISTER_WINDOWS; @@ -1212,9 +1356,23 @@ cont_save_machine_stack(rb_thread_t *th, rb_context_t *cont) MEMCPY(cont->machine.stack, cont->machine.stack_src, VALUE, size); } -static const rb_data_type_t cont_data_type = { +static void +cont_handle_weak_references(void *ptr) +{ + rb_context_t *cont = ptr; + + if (!cont) return; + + if (!rb_gc_handle_weak_references_alive_p(cont->saved_ec.gen_fields_cache.obj) || + !rb_gc_handle_weak_references_alive_p(cont->saved_ec.gen_fields_cache.fields_obj)) { + cont->saved_ec.gen_fields_cache.obj = Qundef; + cont->saved_ec.gen_fields_cache.fields_obj = Qundef; + } +} + +static const rb_data_type_t rb_cont_data_type = { "continuation", - {cont_mark, cont_free, cont_memsize, cont_compact}, + {cont_mark, cont_free, cont_memsize, cont_compact, cont_handle_weak_references}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; @@ -1245,7 +1403,7 @@ jit_cont_new(rb_execution_context_t *ec) // We need to use calloc instead of something like ZALLOC to avoid triggering GC here. // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup, // the thread is still being prepared and marking it causes SEGV. - cont = calloc(1, sizeof(struct rb_jit_cont)); + cont = ruby_mimcalloc(1, sizeof(struct rb_jit_cont)); if (cont == NULL) rb_memerror(); cont->ec = ec; @@ -1284,7 +1442,7 @@ jit_cont_free(struct rb_jit_cont *cont) } rb_native_mutex_unlock(&jit_cont_lock); - free(cont); + ruby_mimfree(cont); } // Call a given callback against all on-stack ISEQs. @@ -1298,8 +1456,11 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) { - callback(cfp->iseq, data); + if (CFP_PC(cfp) && CFP_ISEQ(cfp)) { + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { + callback(iseq, data); + } } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } @@ -1335,7 +1496,7 @@ rb_jit_cont_finish(void) struct rb_jit_cont *cont, *next; for (cont = first_jit_cont; cont != NULL; cont = next) { next = cont->next; - free(cont); // Don't use xfree because it's allocated by calloc. + ruby_mimfree(cont); // Don't use xfree because it's allocated by mimcalloc. } rb_native_mutex_destroy(&jit_cont_lock); } @@ -1374,7 +1535,8 @@ cont_new(VALUE klass) rb_thread_t *th = GET_THREAD(); THREAD_MUST_BE_RUNNING(th); - contval = TypedData_Make_Struct(klass, rb_context_t, &cont_data_type, cont); + contval = TypedData_Make_Struct(klass, rb_context_t, &rb_cont_data_type, cont); + rb_gc_declare_weak_references(contval); cont->self = contval; cont_init(cont, th); return cont; @@ -1418,8 +1580,8 @@ show_vm_pcs(const rb_control_frame_t *cfp, int i=0; while (cfp != end_of_cfp) { int pc = 0; - if (cfp->iseq) { - pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + if (CFP_ISEQ(cfp)) { + pc = cfp->pc - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; } fprintf(stderr, "%2d pc: %d\n", i++, pc); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1443,6 +1605,7 @@ cont_capture(volatile int *volatile stat) #ifdef CAPTURE_JUST_VALID_VM_STACK cont->saved_vm_stack.slen = ec->cfp->sp - ec->vm_stack; cont->saved_vm_stack.clen = ec->vm_stack + ec->vm_stack_size - (VALUE*)ec->cfp; + cont->saved_vm_stack.size = cont->saved_vm_stack.slen + cont->saved_vm_stack.clen; cont->saved_vm_stack.ptr = ALLOC_N(VALUE, cont->saved_vm_stack.slen + cont->saved_vm_stack.clen); MEMCPY(cont->saved_vm_stack.ptr, ec->vm_stack, @@ -1452,6 +1615,7 @@ cont_capture(volatile int *volatile stat) VALUE, cont->saved_vm_stack.clen); #else + cont->saved_vm_stack.size = ec->vm_stack_size; cont->saved_vm_stack.ptr = ALLOC_N(VALUE, ec->vm_stack_size); MEMCPY(cont->saved_vm_stack.ptr, ec->vm_stack, VALUE, ec->vm_stack_size); #endif @@ -1460,22 +1624,6 @@ cont_capture(volatile int *volatile stat) VM_ASSERT(cont->saved_ec.cfp != NULL); cont_save_machine_stack(th, cont); - /* backup ensure_list to array for search in another context */ - { - rb_ensure_list_t *p; - int size = 0; - rb_ensure_entry_t *entry; - for (p=th->ec->ensure_list; p; p=p->next) - size++; - entry = cont->ensure_array = ALLOC_N(rb_ensure_entry_t,size+1); - for (p=th->ec->ensure_list; p; p=p->next) { - if (!p->entry.marker) - p->entry.marker = rb_ary_hidden_new(0); /* dummy object */ - *entry++ = p->entry; - } - entry->marker = 0; - } - if (ruby_setjmp(cont->jmpbuf)) { VALUE value; @@ -1518,6 +1666,51 @@ cont_restore_thread(rb_context_t *cont) rb_raise(rb_eRuntimeError, "can't call across trace_func"); } +#if defined(__wasm__) && !defined(__EMSCRIPTEN__) + if (th->ec->tag != sec->tag) { + /* find the lowest common ancestor tag of the current EC and the saved EC */ + + struct rb_vm_tag *lowest_common_ancestor = NULL; + size_t num_tags = 0; + size_t num_saved_tags = 0; + for (struct rb_vm_tag *tag = th->ec->tag; tag != NULL; tag = tag->prev) { + ++num_tags; + } + for (struct rb_vm_tag *tag = sec->tag; tag != NULL; tag = tag->prev) { + ++num_saved_tags; + } + + size_t min_tags = num_tags <= num_saved_tags ? num_tags : num_saved_tags; + + struct rb_vm_tag *tag = th->ec->tag; + while (num_tags > min_tags) { + tag = tag->prev; + --num_tags; + } + + struct rb_vm_tag *saved_tag = sec->tag; + while (num_saved_tags > min_tags) { + saved_tag = saved_tag->prev; + --num_saved_tags; + } + + while (min_tags > 0) { + if (tag == saved_tag) { + lowest_common_ancestor = tag; + break; + } + tag = tag->prev; + saved_tag = saved_tag->prev; + --min_tags; + } + + /* free all the jump buffers between the current EC's tag and the lowest common ancestor tag */ + for (struct rb_vm_tag *tag = th->ec->tag; tag != lowest_common_ancestor; tag = tag->prev) { + rb_vm_tag_jmpbuf_deinit(&tag->buf); + } + } +#endif + /* copy vm stack */ #ifdef CAPTURE_JUST_VALID_VM_STACK MEMCPY(th->ec->vm_stack, @@ -1536,7 +1729,6 @@ cont_restore_thread(rb_context_t *cont) th->ec->tag = sec->tag; th->ec->root_lep = sec->root_lep; th->ec->root_svar = sec->root_svar; - th->ec->ensure_list = sec->ensure_list; th->ec->errinfo = sec->errinfo; VM_ASSERT(th->ec->vm_stack != NULL); @@ -1568,11 +1760,10 @@ fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber) } } - /* exchange machine_stack_start between old_fiber and new_fiber */ + /* these values are used in rb_gc_mark_machine_context to mark the fiber's stack. */ old_fiber->cont.saved_ec.machine.stack_start = th->ec->machine.stack_start; + old_fiber->cont.saved_ec.machine.stack_end = FIBER_TERMINATED_P(old_fiber) ? NULL : th->ec->machine.stack_end; - /* old_fiber->machine.stack_end should be NULL */ - old_fiber->cont.saved_ec.machine.stack_end = NULL; // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] -> %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base); @@ -1606,9 +1797,9 @@ cont_restore_1(rb_context_t *cont) cont_restore_thread(cont); /* restore machine stack */ -#if defined(_M_AMD64) && !defined(__MINGW64__) +#if (defined(_M_AMD64) && !defined(__MINGW64__)) || defined(_M_ARM64) { - /* workaround for x64 SEH */ + /* workaround for x64 and arm64 SEH on Windows */ jmp_buf buf; setjmp(buf); _JUMP_BUFFER *bp = (void*)&cont->jmpbuf; @@ -1775,6 +1966,13 @@ rb_callcc(VALUE self) return rb_yield(val); } } +#ifdef RUBY_ASAN_ENABLED +/* callcc can't possibly work with ASAN; see bug #20273. Also this function + * definition below avoids a "defined and not used" warning. */ +MAYBE_UNUSED(static void notusing_callcc(void)) { rb_callcc(Qnil); } +# define rb_callcc rb_f_notimplement +#endif + static VALUE make_passing_arg(int argc, const VALUE *argv) @@ -1793,80 +1991,6 @@ make_passing_arg(int argc, const VALUE *argv) typedef VALUE e_proc(VALUE); -/* CAUTION!! : Currently, error in rollback_func is not supported */ -/* same as rb_protect if set rollback_func to NULL */ -void -ruby_register_rollback_func_for_ensure(e_proc *ensure_func, e_proc *rollback_func) -{ - st_table **table_p = &GET_VM()->ensure_rollback_table; - if (UNLIKELY(*table_p == NULL)) { - *table_p = st_init_numtable(); - } - st_insert(*table_p, (st_data_t)ensure_func, (st_data_t)rollback_func); -} - -static inline e_proc * -lookup_rollback_func(e_proc *ensure_func) -{ - st_table *table = GET_VM()->ensure_rollback_table; - st_data_t val; - if (table && st_lookup(table, (st_data_t)ensure_func, &val)) - return (e_proc *) val; - return (e_proc *) Qundef; -} - - -static inline void -rollback_ensure_stack(VALUE self,rb_ensure_list_t *current,rb_ensure_entry_t *target) -{ - rb_ensure_list_t *p; - rb_ensure_entry_t *entry; - size_t i, j; - size_t cur_size; - size_t target_size; - size_t base_point; - e_proc *func; - - cur_size = 0; - for (p=current; p; p=p->next) - cur_size++; - target_size = 0; - for (entry=target; entry->marker; entry++) - target_size++; - - /* search common stack point */ - p = current; - base_point = cur_size; - while (base_point) { - if (target_size >= base_point && - p->entry.marker == target[target_size - base_point].marker) - break; - base_point --; - p = p->next; - } - - /* rollback function check */ - for (i=0; i < target_size - base_point; i++) { - if (!lookup_rollback_func(target[i].e_proc)) { - rb_raise(rb_eRuntimeError, "continuation called from out of critical rb_ensure scope"); - } - } - /* pop ensure stack */ - while (cur_size > base_point) { - /* escape from ensure block */ - (*current->entry.e_proc)(current->entry.data2); - current = current->next; - cur_size--; - } - /* push ensure stack */ - for (j = 0; j < i; j++) { - func = lookup_rollback_func(target[i - j - 1].e_proc); - if (!UNDEF_P((VALUE)func)) { - (*func)(target[i - j - 1].data2); - } - } -} - NORETURN(static VALUE rb_cont_call(int argc, VALUE *argv, VALUE contval)); /* @@ -1898,7 +2022,6 @@ rb_cont_call(int argc, VALUE *argv, VALUE contval) rb_raise(rb_eRuntimeError, "continuation called across fiber"); } } - rollback_ensure_stack(contval, th->ec->ensure_list, cont->ensure_array); cont->argc = argc; cont->value = make_passing_arg(argc, argv); @@ -1975,7 +2098,7 @@ rb_cont_call(int argc, VALUE *argv, VALUE contval) * == Non-blocking Fibers * * The concept of <em>non-blocking fiber</em> was introduced in Ruby 3.0. - * A non-blocking fiber, when reaching a operation that would normally block + * A non-blocking fiber, when reaching an operation that would normally block * the fiber (like <code>sleep</code>, or wait for another process or I/O) * will yield control to other fibers and allow the <em>scheduler</em> to * handle blocking and waking up (resuming) this fiber when it can proceed. @@ -1994,16 +2117,38 @@ rb_cont_call(int argc, VALUE *argv, VALUE contval) * */ -static const rb_data_type_t fiber_data_type = { +static void +fiber_handle_weak_references(void *ptr) +{ + rb_fiber_t *fiber = ptr; + + if (!fiber) return; + + if (!rb_gc_handle_weak_references_alive_p(fiber->cont.saved_ec.gen_fields_cache.obj) || + !rb_gc_handle_weak_references_alive_p(fiber->cont.saved_ec.gen_fields_cache.fields_obj)) { + fiber->cont.saved_ec.gen_fields_cache.obj = Qundef; + fiber->cont.saved_ec.gen_fields_cache.fields_obj = Qundef; + } +} + +static const rb_data_type_t rb_fiber_data_type = { "fiber", - {fiber_mark, fiber_free, fiber_memsize, fiber_compact,}, + {fiber_mark, fiber_free, fiber_memsize, fiber_compact, fiber_handle_weak_references}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; static VALUE fiber_alloc(VALUE klass) { - return TypedData_Wrap_Struct(klass, &fiber_data_type, 0); + VALUE obj = TypedData_Wrap_Struct(klass, &rb_fiber_data_type, 0); + rb_gc_declare_weak_references(obj); + return obj; +} + +static rb_serial_t +next_ec_serial(rb_ractor_t *cr) +{ + return cr->next_ec_serial++; } static rb_fiber_t* @@ -2025,6 +2170,7 @@ fiber_t_alloc(VALUE fiber_value, unsigned int blocking) cont_init(&fiber->cont, th); fiber->cont.saved_ec.fiber_ptr = fiber; + fiber->cont.saved_ec.serial = next_ec_serial(th->ractor); rb_ec_clear_vm_stack(&fiber->cont.saved_ec); fiber->prev = NULL; @@ -2038,32 +2184,10 @@ fiber_t_alloc(VALUE fiber_value, unsigned int blocking) return fiber; } -static rb_fiber_t * -root_fiber_alloc(rb_thread_t *th) -{ - VALUE fiber_value = fiber_alloc(rb_cFiber); - rb_fiber_t *fiber = th->ec->fiber_ptr; - - VM_ASSERT(DATA_PTR(fiber_value) == NULL); - VM_ASSERT(fiber->cont.type == FIBER_CONTEXT); - VM_ASSERT(FIBER_RESUMED_P(fiber)); - - th->root_fiber = fiber; - DATA_PTR(fiber_value) = fiber; - fiber->cont.self = fiber_value; - - coroutine_initialize_main(&fiber->context); - - return fiber; -} - static inline rb_fiber_t* fiber_current(void) { rb_execution_context_t *ec = GET_EC(); - if (ec->fiber_ptr->cont.self == 0) { - root_fiber_alloc(rb_ec_thread_ptr(ec)); - } return ec->fiber_ptr; } @@ -2196,14 +2320,14 @@ rb_fiber_storage_set(VALUE self, VALUE value) * Returns the value of the fiber storage variable identified by +key+. * * The +key+ must be a symbol, and the value is set by Fiber#[]= or - * Fiber#store. + * Fiber#storage. * * See also Fiber::[]=. */ static VALUE rb_fiber_storage_aref(VALUE class, VALUE key) { - Check_Type(key, T_SYMBOL); + key = rb_to_symbol(key); VALUE storage = fiber_storage_get(fiber_current(), FALSE); if (storage == Qnil) return Qnil; @@ -2224,7 +2348,7 @@ rb_fiber_storage_aref(VALUE class, VALUE key) static VALUE rb_fiber_storage_aset(VALUE class, VALUE key, VALUE value) { - Check_Type(key, T_SYMBOL); + key = rb_to_symbol(key); VALUE storage = fiber_storage_get(fiber_current(), value != Qnil); if (storage == Qnil) return Qnil; @@ -2565,22 +2689,36 @@ rb_fiber_start(rb_fiber_t *fiber) void rb_threadptr_root_fiber_setup(rb_thread_t *th) { - rb_fiber_t *fiber = ruby_mimmalloc(sizeof(rb_fiber_t)); + rb_fiber_t *fiber = ZALLOC(rb_fiber_t); if (!fiber) { rb_bug("%s", strerror(errno)); /* ... is it possible to call rb_bug here? */ } - MEMZERO(fiber, rb_fiber_t, 1); + fiber->cont.type = FIBER_CONTEXT; fiber->cont.saved_ec.fiber_ptr = fiber; + fiber->cont.saved_ec.serial = next_ec_serial(th->ractor); fiber->cont.saved_ec.thread_ptr = th; fiber->blocking = 1; fiber->killed = 0; fiber_status_set(fiber, FIBER_RESUMED); /* skip CREATED */ + + coroutine_initialize_main(&fiber->context); + th->ec = &fiber->cont.saved_ec; + cont_init_jit_cont(&fiber->cont); } void +rb_root_fiber_obj_setup(rb_thread_t *th) +{ + rb_fiber_t *fiber = th->ec->fiber_ptr; + VALUE fiber_value = fiber_alloc(rb_cFiber); + DATA_PTR(fiber_value) = fiber; + fiber->cont.self = fiber_value; +} + +void rb_threadptr_root_fiber_release(rb_thread_t *th) { if (th->root_fiber) { @@ -2650,15 +2788,7 @@ rb_fiber_current(void) static inline void fiber_store(rb_fiber_t *next_fiber, rb_thread_t *th) { - rb_fiber_t *fiber; - - if (th->ec->fiber_ptr != NULL) { - fiber = th->ec->fiber_ptr; - } - else { - /* create root fiber */ - fiber = root_fiber_alloc(th); - } + rb_fiber_t *fiber = th->ec->fiber_ptr; if (FIBER_CREATED_P(next_fiber)) { fiber_prepare_stack(next_fiber); @@ -2694,7 +2824,9 @@ fiber_switch(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat, rb_fi rb_thread_t *th = GET_THREAD(); /* make sure the root_fiber object is available */ - if (th->root_fiber == NULL) root_fiber_alloc(th); + if (th->root_fiber == NULL) { + th->root_fiber = th->ec->fiber_ptr; + } if (th->ec->fiber_ptr == fiber) { /* ignore fiber context switch @@ -2759,8 +2891,10 @@ fiber_switch(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat, rb_fi // We cannot free the stack until the pthread is joined: #ifndef COROUTINE_PTHREAD_CONTEXT - if (resuming_fiber && FIBER_TERMINATED_P(fiber)) { - fiber_stack_release(fiber); + if (FIBER_TERMINATED_P(fiber)) { + RB_VM_LOCKING() { + fiber_stack_release(fiber); + } } #endif @@ -2900,6 +3034,7 @@ void rb_fiber_close(rb_fiber_t *fiber) { fiber_status_set(fiber, FIBER_TERMINATED); + rb_ec_close(&fiber->cont.saved_ec); } static void @@ -3223,7 +3358,13 @@ rb_fiber_s_yield(int argc, VALUE *argv, VALUE klass) static VALUE fiber_raise(rb_fiber_t *fiber, VALUE exception) { - if (FIBER_SUSPENDED_P(fiber) && !fiber->yielding) { + if (fiber == fiber_current()) { + rb_exc_raise(exception); + } + else if (fiber->resuming_fiber) { + return fiber_raise(fiber->resuming_fiber, exception); + } + else if (FIBER_SUSPENDED_P(fiber) && !fiber->yielding) { return fiber_transfer_kw(fiber, -1, &exception, RB_NO_KEYWORDS); } else { @@ -3232,35 +3373,46 @@ fiber_raise(rb_fiber_t *fiber, VALUE exception) } VALUE -rb_fiber_raise(VALUE fiber, int argc, const VALUE *argv) +rb_fiber_raise(VALUE fiber, int argc, VALUE *argv) { - VALUE exception = rb_make_exception(argc, argv); + VALUE exception = rb_exception_setup(argc, argv); return fiber_raise(fiber_ptr(fiber), exception); } /* * call-seq: - * fiber.raise -> obj - * fiber.raise(string) -> obj - * fiber.raise(exception [, string [, array]]) -> obj + * raise(exception, message = exception.to_s, backtrace = nil, cause: $!) + * raise(message = nil, cause: $!) * * Raises an exception in the fiber at the point at which the last - * +Fiber.yield+ was called. If the fiber has not been started or has + * +Fiber.yield+ was called. + * + * f = Fiber.new { + * puts "Before the yield" + * Fiber.yield 1 # -- exception will be raised here + * puts "After the yield" + * } + * + * p f.resume + * f.raise "Gotcha" + * + * Output + * + * Before the first yield + * 1 + * t.rb:8:in 'Fiber.yield': Gotcha (RuntimeError) + * from t.rb:8:in 'block in <main>' + * + * If the fiber has not been started or has * already run to completion, raises +FiberError+. If the fiber is * yielding, it is resumed. If it is transferring, it is transferred into. * But if it is resuming, raises +FiberError+. * - * With no arguments, raises a +RuntimeError+. With a single +String+ - * argument, raises a +RuntimeError+ with the string as a message. Otherwise, - * the first parameter should be the name of an +Exception+ class (or an - * object that returns an +Exception+ object when sent an +exception+ - * message). The optional second parameter sets the message associated with - * the exception, and the third parameter is an array of callback information. - * Exceptions are caught by the +rescue+ clause of <code>begin...end</code> - * blocks. - * * Raises +FiberError+ if called on a Fiber belonging to another +Thread+. + * + * See Kernel#raise for more information on arguments. + * */ static VALUE rb_fiber_m_raise(int argc, VALUE *argv, VALUE self) @@ -3299,7 +3451,8 @@ rb_fiber_m_kill(VALUE self) else if (fiber->status != FIBER_TERMINATED) { if (fiber_current() == fiber) { fiber_check_killed(fiber); - } else { + } + else { fiber_raise(fiber_ptr(self), Qnil); } } @@ -3354,6 +3507,8 @@ rb_fiber_atfork(rb_thread_t *th) th->root_fiber = th->ec->fiber_ptr; } th->root_fiber->prev = 0; + th->root_fiber->blocking = 1; + th->blocking = 1; } } #endif @@ -3366,7 +3521,7 @@ fiber_pool_free(void *ptr) RUBY_FREE_ENTER("fiber_pool"); fiber_pool_allocation_free(fiber_pool->allocations); - ruby_xfree(fiber_pool); + SIZED_FREE(fiber_pool); RUBY_FREE_LEAVE("fiber_pool"); } @@ -3420,7 +3575,7 @@ rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self) TypedData_Get_Struct(self, struct fiber_pool, &FiberPoolDataType, fiber_pool); - fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), NUM2SIZET(vm_stack_size)); + fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), 0, NUM2SIZET(vm_stack_size)); return self; } @@ -3439,6 +3594,46 @@ rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self) * fiber.resume #=> FiberError: dead fiber called */ +static size_t +shared_fiber_pool_minimum_count(void) +{ + size_t minimum_count = FIBER_POOL_MINIMUM_COUNT; + + const char *minimum_count_env = getenv("RUBY_SHARED_FIBER_POOL_MINIMUM_COUNT"); + if (minimum_count_env && minimum_count_env[0]) { + char *end; + unsigned long value = strtoul(minimum_count_env, &end, 10); + if (end != minimum_count_env && *end == '\0') { + minimum_count = (size_t)value; + } + else { + rb_warn("invalid RUBY_SHARED_FIBER_POOL_MINIMUM_COUNT=%s (expected a non-negative integer)", minimum_count_env); + } + } + + return minimum_count; +} + +static size_t +shared_fiber_pool_maximum_count(void) +{ + size_t maximum_count = 0; + + const char *maximum_count_env = getenv("RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT"); + if (maximum_count_env && maximum_count_env[0]) { + char *end; + unsigned long value = strtoul(maximum_count_env, &end, 10); + if (end != maximum_count_env && *end == '\0') { + maximum_count = (size_t)value; + } + else { + rb_warn("invalid RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT=%s (expected a non-negative integer)", maximum_count_env); + } + } + + return maximum_count; +} + void Init_Cont(void) { @@ -3456,7 +3651,11 @@ Init_Cont(void) #endif SET_MACHINE_STACK_END(&th->ec->machine.stack_end); - fiber_pool_initialize(&shared_fiber_pool, stack_size, FIBER_POOL_INITIAL_SIZE, vm_stack_size); + rb_eFiberError = rb_define_class("FiberError", rb_eStandardError); + + size_t minimum_count = shared_fiber_pool_minimum_count(); + size_t maximum_count = shared_fiber_pool_maximum_count(); + fiber_pool_initialize(&shared_fiber_pool, stack_size, minimum_count, maximum_count, vm_stack_size); fiber_initialize_keywords[0] = rb_intern_const("blocking"); fiber_initialize_keywords[1] = rb_intern_const("pool"); @@ -3478,7 +3677,6 @@ Init_Cont(void) rb_cFiber = rb_define_class("Fiber", rb_cObject); rb_define_alloc_func(rb_cFiber, fiber_alloc); - rb_eFiberError = rb_define_class("FiberError", rb_eStandardError); rb_define_singleton_method(rb_cFiber, "yield", rb_fiber_s_yield, -1); rb_define_singleton_method(rb_cFiber, "current", rb_fiber_s_current, 0); rb_define_singleton_method(rb_cFiber, "blocking", rb_fiber_blocking, 0); @@ -3506,7 +3704,15 @@ Init_Cont(void) rb_define_singleton_method(rb_cFiber, "schedule", rb_fiber_s_schedule, -1); + rb_thread_t *current_thread = rb_current_thread(); + RUBY_ASSERT(CLASS_OF(current_thread->ec->fiber_ptr->cont.self) == 0); + *(VALUE *)&((struct RBasic *)current_thread->ec->fiber_ptr->cont.self)->klass = rb_cFiber; + #ifdef RB_EXPERIMENTAL_FIBER_POOL + /* + * Document-class: Fiber::Pool + * :nodoc: experimental + */ rb_cFiberPool = rb_define_class_under(rb_cFiber, "Pool", rb_cObject); rb_define_alloc_func(rb_cFiberPool, fiber_pool_alloc); rb_define_method(rb_cFiberPool, "initialize", rb_fiber_pool_initialize, -1); |