diff options
Diffstat (limited to 'gc.c')
| -rw-r--r-- | gc.c | 6791 |
1 files changed, 5459 insertions, 1332 deletions
@@ -3,1728 +3,5855 @@ gc.c - $Author$ - $Date$ created at: Tue Oct 5 09:44:46 JST 1993 - Copyright (C) 1993-2003 Yukihiro Matsumoto + Copyright (C) 1993-2007 Yukihiro Matsumoto Copyright (C) 2000 Network Applied Communication Laboratory, Inc. Copyright (C) 2000 Information-technology Promotion Agency, Japan **********************************************************************/ -#include "ruby.h" -#include "rubysig.h" -#include "st.h" -#include "node.h" -#include "env.h" -#include "re.h" +#include "ruby/internal/config.h" +#ifdef _WIN32 +# include "ruby/ruby.h" +#endif + +#if defined(__wasm__) && !defined(__EMSCRIPTEN__) +# include "wasm/setjmp.h" +# include "wasm/machine.h" +#else +# include <setjmp.h> +#endif +#include <stdarg.h> #include <stdio.h> -#include <setjmp.h> -#include <sys/types.h> + +/* MALLOC_HEADERS_BEGIN */ +#ifndef HAVE_MALLOC_USABLE_SIZE +# ifdef _WIN32 +# define HAVE_MALLOC_USABLE_SIZE +# define malloc_usable_size(a) _msize(a) +# elif defined HAVE_MALLOC_SIZE +# define HAVE_MALLOC_USABLE_SIZE +# define malloc_usable_size(a) malloc_size(a) +# endif +#endif + +#ifdef HAVE_MALLOC_USABLE_SIZE +# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER +/* Alternative malloc header is included in ruby/missing.h */ +# elif defined(HAVE_MALLOC_H) +# include <malloc.h> +# elif defined(HAVE_MALLOC_NP_H) +# include <malloc_np.h> +# elif defined(HAVE_MALLOC_MALLOC_H) +# include <malloc/malloc.h> +# endif +#endif + +/* MALLOC_HEADERS_END */ #ifdef HAVE_SYS_TIME_H -#include <sys/time.h> +# include <sys/time.h> #endif #ifdef HAVE_SYS_RESOURCE_H -#include <sys/resource.h> +# include <sys/resource.h> #endif -#ifdef __ia64__ -#include <ucontext.h> -#if defined(__FreeBSD__) -/* - * FreeBSD/ia64 currently does not have a way for a process to get the - * base address for the RSE backing store, so hardcode it. - */ -#define __libc_ia64_register_backing_store_base (4ULL<<61) -#else -#pragma weak __libc_ia64_register_backing_store_base -extern unsigned long __libc_ia64_register_backing_store_base; +#if defined _WIN32 || defined __CYGWIN__ +# include <windows.h> +#elif defined(HAVE_POSIX_MEMALIGN) +#elif defined(HAVE_MEMALIGN) +# include <malloc.h> #endif + +#include <sys/types.h> + +#ifdef __EMSCRIPTEN__ +#include <emscripten.h> +#endif + +/* For ruby_annotate_mmap */ +#ifdef HAVE_SYS_PRCTL_H +#include <sys/prctl.h> #endif -void re_free_registers _((struct re_registers*)); -void rb_io_fptr_finalize _((struct OpenFile*)); +#undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */ + +#include "constant.h" +#include "debug_counter.h" +#include "eval_intern.h" +#include "gc/gc.h" +#include "id_table.h" +#include "internal.h" +#include "internal/class.h" +#include "internal/compile.h" +#include "internal/complex.h" +#include "internal/concurrent_set.h" +#include "internal/cont.h" +#include "internal/error.h" +#include "internal/eval.h" +#include "internal/gc.h" +#include "internal/hash.h" +#include "internal/imemo.h" +#include "internal/io.h" +#include "internal/numeric.h" +#include "internal/object.h" +#include "internal/proc.h" +#include "internal/rational.h" +#include "internal/re.h" +#include "internal/sanitizers.h" +#include "internal/struct.h" +#include "internal/symbol.h" +#include "internal/thread.h" +#include "internal/variable.h" +#include "internal/warnings.h" +#include "probes.h" +#include "regint.h" +#include "ruby/debug.h" +#include "ruby/io.h" +#include "ruby/re.h" +#include "ruby/st.h" +#include "ruby/thread.h" +#include "ruby/util.h" +#include "ruby/vm.h" +#include "ruby_assert.h" +#include "ruby_atomic.h" +#include "symbol.h" +#include "variable.h" +#include "vm_core.h" +#include "vm_sync.h" +#include "vm_callinfo.h" +#include "ractor_core.h" +#include "yjit.h" +#include "zjit.h" + +#include "builtin.h" +#include "shape.h" + +// TODO: Don't export this function in modular GC, instead MMTk should figure out +// how to combine GC thread backtrace with mutator thread backtrace. +void +rb_gc_print_backtrace(void) +{ + rb_print_backtrace(stderr); +} + +unsigned int +rb_gc_vm_lock(const char *file, int line) +{ + unsigned int lev = 0; + rb_vm_lock_enter(&lev, file, line); + return lev; +} + +void +rb_gc_vm_unlock(unsigned int lev, const char *file, int line) +{ + rb_vm_lock_leave(&lev, file, line); +} + +unsigned int +rb_gc_cr_lock(const char *file, int line) +{ + unsigned int lev; + rb_vm_lock_enter_cr(GET_RACTOR(), &lev, file, line); + return lev; +} + +void +rb_gc_cr_unlock(unsigned int lev, const char *file, int line) +{ + rb_vm_lock_leave_cr(GET_RACTOR(), &lev, file, line); +} + +unsigned int +rb_gc_vm_lock_no_barrier(const char *file, int line) +{ + unsigned int lev = 0; + rb_vm_lock_enter_nb(&lev, file, line); + return lev; +} + +void +rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line) +{ + rb_vm_lock_leave_nb(&lev, file, line); +} + +void +rb_gc_vm_barrier(void) +{ + rb_vm_barrier(); +} + +void * +rb_gc_get_ractor_newobj_cache(void) +{ + return GET_RACTOR()->newobj_cache; +} -#if !defined(setjmp) && defined(HAVE__SETJMP) -#define setjmp(env) _setjmp(env) +void +rb_gc_initialize_vm_context(struct rb_gc_vm_context *context) +{ + rb_native_mutex_initialize(&context->lock); + context->ec = GET_EC(); +} + +bool +rb_gc_event_hook_required_p(rb_event_flag_t event) +{ + return ruby_vm_event_flags & event; +} + +void +rb_gc_event_hook(VALUE obj, rb_event_flag_t event) +{ + if (LIKELY(!rb_gc_event_hook_required_p(event))) return; + + rb_execution_context_t *ec = rb_gc_get_ec(); + if (!ec->cfp) return; + +#if USE_MODULAR_GC + bool gc_thread_p = false; + if (!GET_EC()) { + gc_thread_p = true; + +# ifdef RB_THREAD_LOCAL_SPECIFIER + rb_current_ec_set(ec); +# else + native_tls_set(ruby_current_ec_key, ec); +# endif + } #endif -/* Make alloca work the best possible way. */ -#ifdef __GNUC__ -# ifndef atarist -# ifndef alloca -# define alloca __builtin_alloca -# endif -# endif /* atarist */ -#else -# ifdef HAVE_ALLOCA_H -# include <alloca.h> + EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, obj); + +#if USE_MODULAR_GC + if (gc_thread_p) { +# ifdef RB_THREAD_LOCAL_SPECIFIER + rb_current_ec_set(NULL); # else -# ifdef _AIX - #pragma alloca -# else -# ifndef alloca /* predefined by HP cc +Olibcalls */ -void *alloca (); -# endif -# endif /* AIX */ -# endif /* HAVE_ALLOCA_H */ -#endif /* __GNUC__ */ - -#ifndef GC_MALLOC_LIMIT -#if defined(MSDOS) || defined(__human68k__) -#define GC_MALLOC_LIMIT 200000 + native_tls_set(ruby_current_ec_key, NULL); +# endif + } +#endif +} + +void * +rb_gc_get_objspace(void) +{ + return GET_VM()->gc.objspace; +} + +void +rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data) +{ + rb_ractor_t *r = NULL; + if (RB_LIKELY(ruby_single_main_ractor)) { + GC_ASSERT( + ccan_list_empty(&GET_VM()->ractor.set) || + (ccan_list_top(&GET_VM()->ractor.set, rb_ractor_t, vmlr_node) == ruby_single_main_ractor && + ccan_list_tail(&GET_VM()->ractor.set, rb_ractor_t, vmlr_node) == ruby_single_main_ractor) + ); + + func(ruby_single_main_ractor->newobj_cache, data); + } + else { + ccan_list_for_each(&GET_VM()->ractor.set, r, vmlr_node) { + func(r->newobj_cache, data); + } + } +} + +void +rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data) +{ + volatile struct { + VALUE errinfo; + VALUE final; + rb_control_frame_t *cfp; + VALUE *sp; + long finished; + } saved; + + rb_execution_context_t * volatile ec = GET_EC(); +#define RESTORE_FINALIZER() (\ + ec->cfp = saved.cfp, \ + ec->cfp->sp = saved.sp, \ + ec->errinfo = saved.errinfo) + + saved.errinfo = ec->errinfo; + saved.cfp = ec->cfp; + saved.sp = ec->cfp->sp; + saved.finished = 0; + saved.final = Qundef; + + ASSERT_vm_unlocking(); + rb_ractor_ignore_belonging(true); + EC_PUSH_TAG(ec); + enum ruby_tag_type state = EC_EXEC_TAG(); + if (state != TAG_NONE) { + ++saved.finished; /* skip failed finalizer */ + + VALUE failed_final = saved.final; + saved.final = Qundef; + if (!UNDEF_P(failed_final) && !NIL_P(ruby_verbose)) { + rb_warn("Exception in finalizer %+"PRIsVALUE, failed_final); + rb_ec_error_print(ec, ec->errinfo); + } + } + + for (long i = saved.finished; RESTORE_FINALIZER(), i < count; saved.finished = ++i) { + saved.final = callback(i, data); + rb_check_funcall(saved.final, idCall, 1, &objid); + } + EC_POP_TAG(); + rb_ractor_ignore_belonging(false); +#undef RESTORE_FINALIZER +} + +void +rb_gc_set_pending_interrupt(void) +{ + rb_execution_context_t *ec = GET_EC(); + ec->interrupt_mask |= PENDING_INTERRUPT_MASK; +} + +void +rb_gc_unset_pending_interrupt(void) +{ + rb_execution_context_t *ec = GET_EC(); + ec->interrupt_mask &= ~PENDING_INTERRUPT_MASK; +} + +bool +rb_gc_multi_ractor_p(void) +{ + return rb_multi_ractor_p(); +} + +bool +rb_gc_shutdown_call_finalizer_p(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_DATA: + if (!ruby_free_at_exit_p()) { + if (!RDATA(obj)->type) return false; + if (!rbimpl_typeddata_embedded_p(obj) && !RTYPEDDATA(obj)->data) return false; + } + if (rb_obj_is_thread(obj)) return false; + if (rb_obj_is_mutex(obj)) return false; + if (rb_obj_is_fiber(obj)) return false; + if (rb_ractor_p(obj)) return false; + if (rb_obj_is_fstring_table(obj)) return false; + if (rb_obj_is_symbol_table(obj)) return false; + + return true; + + case T_FILE: + return true; + + case T_SYMBOL: + return true; + + case T_NONE: + return false; + + default: + return ruby_free_at_exit_p(); + } +} + +void +rb_gc_obj_changed_pool(VALUE obj, size_t heap_id) +{ + RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT)); + + RBASIC_SET_SHAPE_ID(obj, rb_obj_shape_transition_heap(obj, heap_id)); +} + +void rb_vm_update_references(void *ptr); + +#define rb_setjmp(env) RUBY_SETJMP(env) +#define rb_jmp_buf rb_jmpbuf_t + +#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +#define MAP_ANONYMOUS MAP_ANON +#endif + +#define unless_objspace(objspace) \ + void *objspace; \ + rb_vm_t *unless_objspace_vm = GET_VM(); \ + if (unless_objspace_vm) objspace = unless_objspace_vm->gc.objspace; \ + else /* return; or objspace will be warned uninitialized */ + +#define RMOVED(obj) ((struct RMoved *)(obj)) + +#define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \ + if (gc_object_moved_p_internal((_objspace), (VALUE)(_thing))) { \ + *(_type *)&(_thing) = (_type)gc_location_internal(_objspace, (VALUE)_thing); \ + } \ +} while (0) + +#define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing) + +#if RUBY_MARK_FREE_DEBUG +int ruby_gc_debug_indent = 0; +#endif + +#ifndef RGENGC_OBJ_INFO +# define RGENGC_OBJ_INFO RGENGC_CHECK_MODE +#endif + +#ifndef CALC_EXACT_MALLOC_SIZE +# define CALC_EXACT_MALLOC_SIZE 0 +#endif + +VALUE rb_mGC; + +static size_t malloc_offset = 0; +#if defined(HAVE_MALLOC_USABLE_SIZE) +static size_t +gc_compute_malloc_offset(void) +{ + // Different allocators use different metadata storage strategies which result in different + // ideal sizes. + // For instance malloc(64) will waste 8B with glibc, but waste 0B with jemalloc. + // But malloc(56) will waste 0B with glibc, but waste 8B with jemalloc. + // So we try allocating 64, 56 and 48 bytes and select the first offset that doesn't + // waste memory. + // This was tested on Linux with glibc 2.35 and jemalloc 5, and for both it result in + // no wasted memory. + size_t offset = 0; + for (offset = 0; offset <= 16; offset += 8) { + size_t allocated = (64 - offset); + void *test_ptr = malloc(allocated); + size_t wasted = malloc_usable_size(test_ptr) - allocated; + free(test_ptr); + + if (wasted == 0) { + return offset; + } + } + return 0; +} #else -#define GC_MALLOC_LIMIT 8000000 +static size_t +gc_compute_malloc_offset(void) +{ + // If we don't have malloc_usable_size, we use powers of 2. + return 0; +} #endif + +size_t +rb_malloc_grow_capa(size_t current, size_t type_size) +{ + size_t current_capacity = current; + if (current_capacity < 4) { + current_capacity = 4; + } + current_capacity *= type_size; + + // We double the current capacity. + size_t new_capacity = (current_capacity * 2); + + // And round up to the next power of 2 if it's not already one. + if (rb_popcount64(new_capacity) != 1) { + new_capacity = (size_t)(1 << (64 - nlz_int64(new_capacity))); + } + + new_capacity -= malloc_offset; + new_capacity /= type_size; + if (current > new_capacity) { + rb_bug("rb_malloc_grow_capa: current_capacity=%zu, new_capacity=%zu, malloc_offset=%zu", current, new_capacity, malloc_offset); + } + RUBY_ASSERT(new_capacity > current); + return new_capacity; +} + +static inline struct rbimpl_size_overflow_tag +size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */ +{ + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y); + struct rbimpl_size_overflow_tag u = rbimpl_size_add_overflow(t.result, z); + return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed, u.result }; +} + +static inline struct rbimpl_size_overflow_tag +size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */ +{ + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y); + struct rbimpl_size_overflow_tag u = rbimpl_size_mul_overflow(z, w); + struct rbimpl_size_overflow_tag v = rbimpl_size_add_overflow(t.result, u.result); + return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed || v.overflowed, v.result }; +} + +PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3); + +static inline size_t +size_mul_or_raise(size_t x, size_t y, VALUE exc) +{ + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y); + if (LIKELY(!t.overflowed)) { + return t.result; + } + else if (rb_during_gc()) { + rb_memerror(); /* or...? */ + } + else { + gc_raise( + exc, + "integer overflow: %"PRIuSIZE + " * %"PRIuSIZE + " > %"PRIuSIZE, + x, y, (size_t)SIZE_MAX); + } +} + +size_t +rb_size_mul_or_raise(size_t x, size_t y, VALUE exc) +{ + return size_mul_or_raise(x, y, exc); +} + +static inline size_t +size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc) +{ + struct rbimpl_size_overflow_tag t = size_mul_add_overflow(x, y, z); + if (LIKELY(!t.overflowed)) { + return t.result; + } + else if (rb_during_gc()) { + rb_memerror(); /* or...? */ + } + else { + gc_raise( + exc, + "integer overflow: %"PRIuSIZE + " * %"PRIuSIZE + " + %"PRIuSIZE + " > %"PRIuSIZE, + x, y, z, (size_t)SIZE_MAX); + } +} + +size_t +rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc) +{ + return size_mul_add_or_raise(x, y, z, exc); +} + +static inline size_t +size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc) +{ + struct rbimpl_size_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w); + if (LIKELY(!t.overflowed)) { + return t.result; + } + else if (rb_during_gc()) { + rb_memerror(); /* or...? */ + } + else { + gc_raise( + exc, + "integer overflow: %"PRIdSIZE + " * %"PRIdSIZE + " + %"PRIdSIZE + " * %"PRIdSIZE + " > %"PRIdSIZE, + x, y, z, w, (size_t)SIZE_MAX); + } +} + +#if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL +/* trick the compiler into thinking a external signal handler uses this */ +volatile VALUE rb_gc_guarded_val; +volatile VALUE * +rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val) +{ + rb_gc_guarded_val = val; + + return ptr; +} #endif -static unsigned long malloc_increase = 0; -static unsigned long malloc_limit = GC_MALLOC_LIMIT; -static void run_final(); -static VALUE nomem_error; +static const char *obj_type_name(VALUE obj); +static st_table *id2ref_tbl; +#include "gc/default/default.c" + +#if USE_MODULAR_GC && !defined(HAVE_DLOPEN) +# error "Modular GC requires dlopen" +#elif USE_MODULAR_GC +#include <dlfcn.h> + +typedef struct gc_function_map { + // Bootup + void *(*objspace_alloc)(void); + void (*objspace_init)(void *objspace_ptr); + void *(*ractor_cache_alloc)(void *objspace_ptr, void *ractor); + void (*set_params)(void *objspace_ptr); + void (*init)(void); + size_t *(*heap_sizes)(void *objspace_ptr); + // Shutdown + void (*shutdown_free_objects)(void *objspace_ptr); + void (*objspace_free)(void *objspace_ptr); + void (*ractor_cache_free)(void *objspace_ptr, void *cache); + // GC + void (*start)(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact); + bool (*during_gc_p)(void *objspace_ptr); + void (*prepare_heap)(void *objspace_ptr); + void (*gc_enable)(void *objspace_ptr); + void (*gc_disable)(void *objspace_ptr, bool finish_current_gc); + bool (*gc_enabled_p)(void *objspace_ptr); + VALUE (*config_get)(void *objpace_ptr); + void (*config_set)(void *objspace_ptr, VALUE hash); + void (*stress_set)(void *objspace_ptr, VALUE flag); + VALUE (*stress_get)(void *objspace_ptr); + struct rb_gc_vm_context *(*get_vm_context)(void *objspace_ptr); + // Object allocation + VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size); + size_t (*obj_slot_size)(VALUE obj); + size_t (*heap_id_for_size)(void *objspace_ptr, size_t size); + bool (*size_allocatable_p)(size_t size); + // Malloc + void *(*malloc)(void *objspace_ptr, size_t size, bool gc_allowed); + void *(*calloc)(void *objspace_ptr, size_t size, bool gc_allowed); + void *(*realloc)(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed); + void (*free)(void *objspace_ptr, void *ptr, size_t old_size); + void (*adjust_memory_usage)(void *objspace_ptr, ssize_t diff); + // Marking + void (*mark)(void *objspace_ptr, VALUE obj); + void (*mark_and_move)(void *objspace_ptr, VALUE *ptr); + void (*mark_and_pin)(void *objspace_ptr, VALUE obj); + void (*mark_maybe)(void *objspace_ptr, VALUE obj); + // Weak references + void (*declare_weak_references)(void *objspace_ptr, VALUE obj); + bool (*handle_weak_references_alive_p)(void *objspace_ptr, VALUE obj); + // Compaction + void (*register_pinning_obj)(void *objspace_ptr, VALUE obj); + bool (*object_moved_p)(void *objspace_ptr, VALUE obj); + VALUE (*location)(void *objspace_ptr, VALUE value); + // Write barriers + void (*writebarrier)(void *objspace_ptr, VALUE a, VALUE b); + void (*writebarrier_unprotect)(void *objspace_ptr, VALUE obj); + void (*writebarrier_remember)(void *objspace_ptr, VALUE obj); + // Heap walking + void (*each_objects)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); + void (*each_object)(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data); + // Finalizers + void (*make_zombie)(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data); + VALUE (*define_finalizer)(void *objspace_ptr, VALUE obj, VALUE block); + void (*undefine_finalizer)(void *objspace_ptr, VALUE obj); + void (*copy_finalizer)(void *objspace_ptr, VALUE dest, VALUE obj); + void (*shutdown_call_finalizer)(void *objspace_ptr); + // Forking + void (*before_fork)(void *objspace_ptr); + void (*after_fork)(void *objspace_ptr, rb_pid_t pid); + // Statistics + void (*set_measure_total_time)(void *objspace_ptr, VALUE flag); + bool (*get_measure_total_time)(void *objspace_ptr); + unsigned long long (*get_total_time)(void *objspace_ptr); + size_t (*gc_count)(void *objspace_ptr); + VALUE (*latest_gc_info)(void *objspace_ptr, VALUE key); + VALUE (*stat)(void *objspace_ptr, VALUE hash_or_sym); + VALUE (*stat_heap)(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym); + const char *(*active_gc_name)(void); + // Miscellaneous + struct rb_gc_object_metadata_entry *(*object_metadata)(void *objspace_ptr, VALUE obj); + bool (*pointer_to_heap_p)(void *objspace_ptr, const void *ptr); + bool (*garbage_object_p)(void *objspace_ptr, VALUE obj); + void (*set_event_hook)(void *objspace_ptr, const rb_event_flag_t event); + void (*copy_attributes)(void *objspace_ptr, VALUE dest, VALUE obj); + + bool modular_gc_loaded_p; +} rb_gc_function_map_t; + +static rb_gc_function_map_t rb_gc_functions; + +# define RUBY_GC_LIBRARY "RUBY_GC_LIBRARY" +# define MODULAR_GC_DIR STRINGIZE(modular_gc_dir) -void -rb_memerror() +static void +ruby_modular_gc_init(void) { - static int recurse = 0; + // Assert that the directory path ends with a / + RUBY_ASSERT_ALWAYS(MODULAR_GC_DIR[sizeof(MODULAR_GC_DIR) - 2] == '/'); + + const char *gc_so_file = getenv(RUBY_GC_LIBRARY); + + rb_gc_function_map_t gc_functions = { 0 }; + + char *gc_so_path = NULL; + void *handle = NULL; + if (gc_so_file) { + /* Check to make sure that gc_so_file matches /[\w-_]+/ so that it does + * not load a shared object outside of the directory. */ + for (size_t i = 0; i < strlen(gc_so_file); i++) { + char c = gc_so_file[i]; + if (isalnum(c)) continue; + switch (c) { + case '-': + case '_': + break; + default: + fprintf(stderr, "Only alphanumeric, dash, and underscore is allowed in "RUBY_GC_LIBRARY"\n"); + exit(EXIT_FAILURE); + } + } + + size_t gc_so_path_size = strlen(MODULAR_GC_DIR "librubygc." DLEXT) + strlen(gc_so_file) + 1; +#ifdef LOAD_RELATIVE + Dl_info dli; + size_t prefix_len = 0; + if (dladdr((void *)(uintptr_t)ruby_modular_gc_init, &dli)) { + const char *base = strrchr(dli.dli_fname, '/'); + if (base) { + size_t tail = 0; +# define end_with_p(lit) \ + (prefix_len >= (tail = rb_strlen_lit(lit)) && \ + memcmp(base - tail, lit, tail) == 0) + + prefix_len = base - dli.dli_fname; + if (end_with_p("/bin") || end_with_p("/lib")) { + prefix_len -= tail; + } + prefix_len += MODULAR_GC_DIR[0] != '/'; + gc_so_path_size += prefix_len; + } + } +#endif + gc_so_path = alloca(gc_so_path_size); + { + size_t gc_so_path_idx = 0; +#define GC_SO_PATH_APPEND(str) do { \ + gc_so_path_idx += strlcpy(gc_so_path + gc_so_path_idx, str, gc_so_path_size - gc_so_path_idx); \ +} while (0) +#ifdef LOAD_RELATIVE + if (prefix_len > 0) { + memcpy(gc_so_path, dli.dli_fname, prefix_len); + gc_so_path_idx = prefix_len; + } +#endif + GC_SO_PATH_APPEND(MODULAR_GC_DIR "librubygc."); + GC_SO_PATH_APPEND(gc_so_file); + GC_SO_PATH_APPEND(DLEXT); + GC_ASSERT(gc_so_path_idx == gc_so_path_size - 1); +#undef GC_SO_PATH_APPEND + } + + handle = dlopen(gc_so_path, RTLD_LAZY | RTLD_GLOBAL); + if (!handle) { + fprintf(stderr, "ruby_modular_gc_init: Shared library %s cannot be opened: %s\n", gc_so_path, dlerror()); + exit(EXIT_FAILURE); + } + + gc_functions.modular_gc_loaded_p = true; + } + + unsigned int err_count = 0; + +# define load_modular_gc_func(name) do { \ + if (handle) { \ + const char *func_name = "rb_gc_impl_" #name; \ + gc_functions.name = dlsym(handle, func_name); \ + if (!gc_functions.name) { \ + fprintf(stderr, "ruby_modular_gc_init: %s function not exported by library %s\n", func_name, gc_so_path); \ + err_count++; \ + } \ + } \ + else { \ + gc_functions.name = rb_gc_impl_##name; \ + } \ +} while (0) - if (recurse > 0 && rb_safe_level() < 4) { - fprintf(stderr, "[FATAL] failed to allocate memory\n"); - exit(1); + // Bootup + load_modular_gc_func(objspace_alloc); + load_modular_gc_func(objspace_init); + load_modular_gc_func(ractor_cache_alloc); + load_modular_gc_func(set_params); + load_modular_gc_func(init); + load_modular_gc_func(heap_sizes); + // Shutdown + load_modular_gc_func(shutdown_free_objects); + load_modular_gc_func(objspace_free); + load_modular_gc_func(ractor_cache_free); + // GC + load_modular_gc_func(start); + load_modular_gc_func(during_gc_p); + load_modular_gc_func(prepare_heap); + load_modular_gc_func(gc_enable); + load_modular_gc_func(gc_disable); + load_modular_gc_func(gc_enabled_p); + load_modular_gc_func(config_set); + load_modular_gc_func(config_get); + load_modular_gc_func(stress_set); + load_modular_gc_func(stress_get); + load_modular_gc_func(get_vm_context); + // Object allocation + load_modular_gc_func(new_obj); + load_modular_gc_func(obj_slot_size); + load_modular_gc_func(heap_id_for_size); + load_modular_gc_func(size_allocatable_p); + // Malloc + load_modular_gc_func(malloc); + load_modular_gc_func(calloc); + load_modular_gc_func(realloc); + load_modular_gc_func(free); + load_modular_gc_func(adjust_memory_usage); + // Marking + load_modular_gc_func(mark); + load_modular_gc_func(mark_and_move); + load_modular_gc_func(mark_and_pin); + load_modular_gc_func(mark_maybe); + // Weak references + load_modular_gc_func(declare_weak_references); + load_modular_gc_func(handle_weak_references_alive_p); + // Compaction + load_modular_gc_func(register_pinning_obj); + load_modular_gc_func(object_moved_p); + load_modular_gc_func(location); + // Write barriers + load_modular_gc_func(writebarrier); + load_modular_gc_func(writebarrier_unprotect); + load_modular_gc_func(writebarrier_remember); + // Heap walking + load_modular_gc_func(each_objects); + load_modular_gc_func(each_object); + // Finalizers + load_modular_gc_func(make_zombie); + load_modular_gc_func(define_finalizer); + load_modular_gc_func(undefine_finalizer); + load_modular_gc_func(copy_finalizer); + load_modular_gc_func(shutdown_call_finalizer); + // Forking + load_modular_gc_func(before_fork); + load_modular_gc_func(after_fork); + // Statistics + load_modular_gc_func(set_measure_total_time); + load_modular_gc_func(get_measure_total_time); + load_modular_gc_func(get_total_time); + load_modular_gc_func(gc_count); + load_modular_gc_func(latest_gc_info); + load_modular_gc_func(stat); + load_modular_gc_func(stat_heap); + load_modular_gc_func(active_gc_name); + // Miscellaneous + load_modular_gc_func(object_metadata); + load_modular_gc_func(pointer_to_heap_p); + load_modular_gc_func(garbage_object_p); + load_modular_gc_func(set_event_hook); + load_modular_gc_func(copy_attributes); + + if (err_count > 0) { + fprintf(stderr, "ruby_modular_gc_init: found %u missing exports in library %s\n", err_count, gc_so_path); + exit(EXIT_FAILURE); } - recurse++; - rb_exc_raise(nomem_error); + +# undef load_modular_gc_func + + rb_gc_functions = gc_functions; } +// Bootup +# define rb_gc_impl_objspace_alloc rb_gc_functions.objspace_alloc +# define rb_gc_impl_objspace_init rb_gc_functions.objspace_init +# define rb_gc_impl_ractor_cache_alloc rb_gc_functions.ractor_cache_alloc +# define rb_gc_impl_set_params rb_gc_functions.set_params +# define rb_gc_impl_init rb_gc_functions.init +# define rb_gc_impl_heap_sizes rb_gc_functions.heap_sizes +// Shutdown +# define rb_gc_impl_shutdown_free_objects rb_gc_functions.shutdown_free_objects +# define rb_gc_impl_objspace_free rb_gc_functions.objspace_free +# define rb_gc_impl_ractor_cache_free rb_gc_functions.ractor_cache_free +// GC +# define rb_gc_impl_start rb_gc_functions.start +# define rb_gc_impl_during_gc_p rb_gc_functions.during_gc_p +# define rb_gc_impl_prepare_heap rb_gc_functions.prepare_heap +# define rb_gc_impl_gc_enable rb_gc_functions.gc_enable +# define rb_gc_impl_gc_disable rb_gc_functions.gc_disable +# define rb_gc_impl_gc_enabled_p rb_gc_functions.gc_enabled_p +# define rb_gc_impl_config_get rb_gc_functions.config_get +# define rb_gc_impl_config_set rb_gc_functions.config_set +# define rb_gc_impl_stress_set rb_gc_functions.stress_set +# define rb_gc_impl_stress_get rb_gc_functions.stress_get +# define rb_gc_impl_get_vm_context rb_gc_functions.get_vm_context +// Object allocation +# define rb_gc_impl_new_obj rb_gc_functions.new_obj +# define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size +# define rb_gc_impl_heap_id_for_size rb_gc_functions.heap_id_for_size +# define rb_gc_impl_size_allocatable_p rb_gc_functions.size_allocatable_p +// Malloc +# define rb_gc_impl_malloc rb_gc_functions.malloc +# define rb_gc_impl_calloc rb_gc_functions.calloc +# define rb_gc_impl_realloc rb_gc_functions.realloc +# define rb_gc_impl_free rb_gc_functions.free +# define rb_gc_impl_adjust_memory_usage rb_gc_functions.adjust_memory_usage +// Marking +# define rb_gc_impl_mark rb_gc_functions.mark +# define rb_gc_impl_mark_and_move rb_gc_functions.mark_and_move +# define rb_gc_impl_mark_and_pin rb_gc_functions.mark_and_pin +# define rb_gc_impl_mark_maybe rb_gc_functions.mark_maybe +// Weak references +# define rb_gc_impl_declare_weak_references rb_gc_functions.declare_weak_references +# define rb_gc_impl_handle_weak_references_alive_p rb_gc_functions.handle_weak_references_alive_p +// Compaction +# define rb_gc_impl_register_pinning_obj rb_gc_functions.register_pinning_obj +# define rb_gc_impl_object_moved_p rb_gc_functions.object_moved_p +# define rb_gc_impl_location rb_gc_functions.location +// Write barriers +# define rb_gc_impl_writebarrier rb_gc_functions.writebarrier +# define rb_gc_impl_writebarrier_unprotect rb_gc_functions.writebarrier_unprotect +# define rb_gc_impl_writebarrier_remember rb_gc_functions.writebarrier_remember +// Heap walking +# define rb_gc_impl_each_objects rb_gc_functions.each_objects +# define rb_gc_impl_each_object rb_gc_functions.each_object +// Finalizers +# define rb_gc_impl_make_zombie rb_gc_functions.make_zombie +# define rb_gc_impl_define_finalizer rb_gc_functions.define_finalizer +# define rb_gc_impl_undefine_finalizer rb_gc_functions.undefine_finalizer +# define rb_gc_impl_copy_finalizer rb_gc_functions.copy_finalizer +# define rb_gc_impl_shutdown_call_finalizer rb_gc_functions.shutdown_call_finalizer +// Forking +# define rb_gc_impl_before_fork rb_gc_functions.before_fork +# define rb_gc_impl_after_fork rb_gc_functions.after_fork +// Statistics +# define rb_gc_impl_set_measure_total_time rb_gc_functions.set_measure_total_time +# define rb_gc_impl_get_measure_total_time rb_gc_functions.get_measure_total_time +# define rb_gc_impl_get_total_time rb_gc_functions.get_total_time +# define rb_gc_impl_gc_count rb_gc_functions.gc_count +# define rb_gc_impl_latest_gc_info rb_gc_functions.latest_gc_info +# define rb_gc_impl_stat rb_gc_functions.stat +# define rb_gc_impl_stat_heap rb_gc_functions.stat_heap +# define rb_gc_impl_active_gc_name rb_gc_functions.active_gc_name +// Miscellaneous +# define rb_gc_impl_object_metadata rb_gc_functions.object_metadata +# define rb_gc_impl_pointer_to_heap_p rb_gc_functions.pointer_to_heap_p +# define rb_gc_impl_garbage_object_p rb_gc_functions.garbage_object_p +# define rb_gc_impl_set_event_hook rb_gc_functions.set_event_hook +# define rb_gc_impl_copy_attributes rb_gc_functions.copy_attributes +#endif + +#ifdef RUBY_ASAN_ENABLED +static void +asan_death_callback(void) +{ + if (GET_VM()) { + rb_bug_without_die("ASAN error"); + } +} +#endif + +static VALUE initial_stress = Qfalse; + void * -ruby_xmalloc(size) - long size; +rb_objspace_alloc(void) { - void *mem; +#if USE_MODULAR_GC + ruby_modular_gc_init(); +#endif + + void *objspace = rb_gc_impl_objspace_alloc(); + ruby_current_vm_ptr->gc.objspace = objspace; + rb_gc_impl_objspace_init(objspace); + rb_gc_impl_stress_set(objspace, initial_stress); - if (size < 0) { - rb_raise(rb_eNoMemError, "negative allocation size (or too big)"); +#ifdef RUBY_ASAN_ENABLED + __sanitizer_set_death_callback(asan_death_callback); +#endif + + return objspace; +} + +void +rb_objspace_free(void *objspace) +{ + rb_gc_impl_objspace_free(objspace); +} + +size_t +rb_gc_obj_slot_size(VALUE obj) +{ + return rb_gc_impl_obj_slot_size(obj); +} + +static inline void +gc_validate_pc(VALUE obj) +{ +#if RUBY_DEBUG + // IMEMOs and objects without a class (e.g managed id table) are not traceable + if (RB_TYPE_P(obj, T_IMEMO) || !CLASS_OF(obj)) return; + + rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = ec->cfp; + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && CFP_PC(cfp)) { + const VALUE *iseq_encoded = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(CFP_ISEQ(cfp))->iseq_size; + RUBY_ASSERT(CFP_PC(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(CFP_PC(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } - if (size == 0) size = 1; - malloc_increase += size; +#endif +} - if (malloc_increase > malloc_limit) { - rb_gc(); +NOINLINE(static void gc_newobj_hook(VALUE obj)); +static void +gc_newobj_hook(VALUE obj) +{ + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + size_t slot_size = rb_gc_obj_slot_size(obj); + memset((char *)obj + sizeof(struct RBasic), 0, slot_size - sizeof(struct RBasic)); + + /* We must disable GC here because the callback could call xmalloc + * which could potentially trigger a GC, and a lot of code is unsafe + * to trigger a GC right after an object has been allocated because + * they perform initialization for the object and assume that the + * GC does not trigger before then. */ + bool gc_disabled = RTEST(rb_gc_disable_no_rest()); + { + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_NEWOBJ); + } + if (!gc_disabled) rb_gc_enable(); } - RUBY_CRITICAL(mem = malloc(size)); - if (!mem) { - rb_gc(); - RUBY_CRITICAL(mem = malloc(size)); - if (!mem) { - rb_memerror(); - } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +VALUE +rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size) +{ + GC_ASSERT((flags & FL_WB_PROTECTED) == 0); + rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + VALUE obj = rb_gc_impl_new_obj(rb_gc_get_objspace(), cr->newobj_cache, klass, flags, wb_protected, size); + +#if RACTOR_CHECK_MODE + void rb_ractor_setup_belonging(VALUE obj); + rb_ractor_setup_belonging(obj); +#endif + + RBASIC_SET_SHAPE_ID_NO_CHECKS(obj, shape_id); + + gc_validate_pc(obj); + + if (UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ))) { + gc_newobj_hook(obj); } - return mem; +#if RGENGC_CHECK_MODE +# ifndef GC_DEBUG_SLOT_FILL_SPECIAL_VALUE +# define GC_DEBUG_SLOT_FILL_SPECIAL_VALUE 255 +# endif + + memset( + (void *)(obj + sizeof(struct RBasic)), + GC_DEBUG_SLOT_FILL_SPECIAL_VALUE, + rb_gc_obj_slot_size(obj) - sizeof(struct RBasic) + ); +#endif + + return obj; } -void * -ruby_xcalloc(n, size) - long n, size; +VALUE +rb_ec_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size) { - void *mem; + VALUE type = flags & T_MASK; + RUBY_ASSERT(type != T_OBJECT); + RUBY_ASSERT(type != T_DATA); + RUBY_ASSERT(type != T_CLASS); + RUBY_ASSERT(type != T_MODULE); + RUBY_ASSERT(type != T_ICLASS); + (void)type; + + return rb_newobj(ec, klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size); +} - mem = xmalloc(n * size); - memset(mem, 0, n * size); +VALUE +rb_newobj_of_with_shape(VALUE klass, VALUE flags, shape_id_t shape_id, size_t size) +{ + return rb_newobj(GET_EC(), klass, flags, shape_id, true, size); +} - return mem; +VALUE +rb_newobj_of(VALUE klass, VALUE flags, size_t size) +{ + return rb_newobj(GET_EC(), klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size); } -void * -ruby_xrealloc(ptr, size) - void *ptr; - long size; +static +VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity) { - void *mem; + shape_id_t initial_shape_id = rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(sizeof(struct RObject)))); + VALUE obj = rb_newobj_of_with_shape(klass, T_OBJECT, initial_shape_id, sizeof(struct RObject)); + rb_obj_init_complex(obj, rb_st_init_numtable_with_size(capacity)); + return obj; +} + +VALUE +rb_class_allocate_instance(VALUE klass) +{ + uint32_t index_tbl_num_entries = RCLASS_MAX_IV_COUNT(klass); + VALUE obj; - if (size < 0) { - rb_raise(rb_eArgError, "negative re-allocation size"); + // Directly start as COMPLEX if we know we're over the limit. + RUBY_ASSERT(rb_shape_tree.max_capacity > 0); + if (RB_UNLIKELY(index_tbl_num_entries > rb_shape_tree.max_capacity)) { + obj = class_allocate_complex_instance(klass, index_tbl_num_entries); } - if (!ptr) return xmalloc(size); - if (size == 0) size = 1; - malloc_increase += size; - RUBY_CRITICAL(mem = realloc(ptr, size)); - if (!mem) { - rb_gc(); - RUBY_CRITICAL(mem = realloc(ptr, size)); - if (!mem) { - rb_memerror(); + else { + size_t size = rb_obj_embedded_size(index_tbl_num_entries); + if (!rb_gc_size_allocatable_p(size)) { + size = sizeof(struct RObject); } + + // There might be a NEWOBJ tracepoint callback, and it may set fields. + // So the shape must be passed to `NEWOBJ_OF`. + obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(size))), size); + + #if RUBY_DEBUG + VALUE *ptr = ROBJECT_FIELDS(obj); + size_t fields_count = RSHAPE_LEN(RBASIC_SHAPE_ID(obj)); + for (size_t i = fields_count; i < ROBJECT_FIELDS_CAPACITY(obj); i++) { + ptr[i] = Qundef; + } + #endif } - return mem; +#if RUBY_DEBUG + if (rb_obj_class(obj) != rb_class_real(klass)) { + rb_bug("Expected rb_class_allocate_instance to set the class correctly"); + } +#endif + + return obj; } void -ruby_xfree(x) - void *x; +rb_gc_register_pinning_obj(VALUE obj) { - if (x) - RUBY_CRITICAL(free(x)); + rb_gc_impl_register_pinning_obj(rb_gc_get_objspace(), obj); } -extern int ruby_in_compile; -static int dont_gc; -static int during_gc; -static int need_call_final = 0; -static st_table *finalizer_table = 0; +#define UNEXPECTED_NODE(func) \ + rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \ + BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags) + +static inline void +rb_data_object_check(VALUE klass) +{ + RUBY_ASSERT(!RCLASS_SINGLETON_P(klass)); + if (klass != rb_cObject && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) { + rb_undef_alloc_func(klass); + rb_warn("undefining the allocator of T_DATA class %"PRIsVALUE, klass); + } +} + +#define RTYPEDDATA_EMBEDDED_P rbimpl_typeddata_embedded_p +#define RB_DATA_TYPE_EMBEDDABLE_P(type) ((type)->flags & RUBY_TYPED_EMBEDDABLE) +#define RTYPEDDATA_EMBEDDABLE_P(obj) RB_DATA_TYPE_EMBEDDABLE_P(RTYPEDDATA_TYPE(obj)) + +static VALUE +typed_data_alloc(VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_t *type, size_t size) +{ + RBIMPL_NONNULL_ARG(type); + if (klass) rb_data_object_check(klass); + bool wb_protected = (type->flags & RUBY_FL_WB_PROTECTED) || !type->function.dmark; + VALUE obj = rb_newobj(GET_EC(), klass, T_DATA, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_RDATA, wb_protected, size); + + rb_gc_register_pinning_obj(obj); + + struct RTypedData *data = (struct RTypedData *)obj; + data->fields_obj = 0; + *(VALUE *)&data->type = ((VALUE)type) | typed_flag; + data->data = datap; + + return obj; +} VALUE -rb_gc_enable() +rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type) { - int old = dont_gc; + if (UNLIKELY(RB_DATA_TYPE_EMBEDDABLE_P(type))) { + rb_raise(rb_eTypeError, "Cannot wrap an embeddable TypedData"); + } - dont_gc = Qfalse; - return old; + return typed_data_alloc(klass, 0, datap, type, sizeof(struct RTypedData)); } VALUE -rb_gc_disable() +rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type) { - int old = dont_gc; + if (RB_DATA_TYPE_EMBEDDABLE_P(type)) { + if (!(type->flags & RUBY_TYPED_FREE_IMMEDIATELY)) { + rb_raise(rb_eTypeError, "Embeddable TypedData must be freed immediately"); + } + + size_t embed_size = offsetof(struct RTypedData, data) + size; + if (rb_gc_size_allocatable_p(embed_size)) { + VALUE obj = typed_data_alloc(klass, TYPED_DATA_EMBEDDED, 0, type, embed_size); + memset((char *)obj + offsetof(struct RTypedData, data), 0, size); + return obj; + } + } - dont_gc = Qtrue; - return old; + VALUE obj = typed_data_alloc(klass, 0, NULL, type, sizeof(struct RTypedData)); + DATA_PTR(obj) = xcalloc(1, size); + return obj; } -VALUE rb_mGC; +static size_t +ruby_xmalloc_usable_size(void *ptr) +{ +#ifdef HAVE_MALLOC_USABLE_SIZE +#if CALC_EXACT_MALLOC_SIZE + struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; + return malloc_usable_size(info) - sizeof(struct malloc_obj_info); +#else + return malloc_usable_size(ptr); +#endif +#else + return 0; +#endif +} + +static size_t +rb_objspace_data_type_memsize(VALUE obj) +{ + size_t size = 0; + const void *ptr = RTYPEDDATA_GET_DATA(obj); + + if (ptr) { + if (RTYPEDDATA_EMBEDDABLE_P(obj) && !RTYPEDDATA_EMBEDDED_P(obj)) { + size += ruby_xmalloc_usable_size((void *)ptr); + } + + const rb_data_type_t *type = RTYPEDDATA_TYPE(obj); + if (type->function.dsize) { + size += type->function.dsize(ptr); + } + } -static struct gc_list { - VALUE *varptr; - struct gc_list *next; -} *global_List = 0; + return size; +} + +const char * +rb_objspace_data_type_name(VALUE obj) +{ + return RTYPEDDATA_TYPE(obj)->wrap_struct_name; +} void -rb_gc_register_address(addr) - VALUE *addr; +rb_gc_declare_weak_references(VALUE obj) { - struct gc_list *tmp; + rb_gc_impl_declare_weak_references(rb_gc_get_objspace(), obj); +} + +bool +rb_gc_handle_weak_references_alive_p(VALUE obj) +{ + if (SPECIAL_CONST_P(obj)) return true; - tmp = ALLOC(struct gc_list); - tmp->next = global_List; - tmp->varptr = addr; - global_List = tmp; + return rb_gc_impl_handle_weak_references_alive_p(rb_gc_get_objspace(), obj); } void -rb_gc_unregister_address(addr) - VALUE *addr; +rb_gc_handle_weak_references(VALUE obj) { - struct gc_list *tmp = global_List; + switch (BUILTIN_TYPE(obj)) { + case T_DATA: + { + const rb_data_type_t *type = RTYPEDDATA_TYPE(obj); + + if (type->function.handle_weak_references) { + (type->function.handle_weak_references)(RTYPEDDATA_GET_DATA(obj)); + } + else { + rb_bug( + "rb_gc_handle_weak_references: TypedData %s does not implement handle_weak_references", + RTYPEDDATA_TYPE(obj)->wrap_struct_name + ); + } + } + break; + + case T_IMEMO: { + switch (imemo_type(obj)) { + case imemo_callcache: { + struct rb_callcache *cc = (struct rb_callcache *)obj; + if (cc->klass != Qundef && + (!rb_gc_handle_weak_references_alive_p(cc->klass) || + !rb_gc_handle_weak_references_alive_p((VALUE)cc->cme_))) { + vm_cc_invalidate(cc); + } + break; + } + case imemo_subclasses: { + struct rb_subclasses *subs = (struct rb_subclasses *)obj; + VALUE *entries = rb_imemo_subclasses_entries(obj); + for (uint32_t i = 0; i < subs->count; i++) { + if (entries[i] && !rb_gc_handle_weak_references_alive_p(entries[i])) { + entries[i] = 0; + } + } + break; + } + default: + rb_bug("rb_gc_handle_weak_references: unexpected imemo type"); + } - if (tmp->varptr == addr) { - global_List = tmp->next; - RUBY_CRITICAL(free(tmp)); - return; + break; + } + default: + rb_bug("rb_gc_handle_weak_references: type not supported\n"); } - while (tmp->next) { - if (tmp->next->varptr == addr) { - struct gc_list *t = tmp->next; +} - tmp->next = tmp->next->next; - RUBY_CRITICAL(free(t)); - break; - } - tmp = tmp->next; +static inline bool +rb_gc_imemo_needs_cleanup_p(VALUE obj) +{ + switch (imemo_type(obj)) { + case imemo_constcache: + case imemo_cref: + case imemo_ifunc: + case imemo_memo: + case imemo_svar: + case imemo_callcache: + case imemo_throw_data: + case imemo_cvar_entry: + return false; + + case imemo_env: + case imemo_ment: + case imemo_iseq: + case imemo_callinfo: + case imemo_cdhash: + return true; + + case imemo_subclasses: + return FL_TEST_RAW(obj, IMEMO_SUBCLASSES_HEAP); + + case imemo_tmpbuf: + return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL; + + case imemo_fields: + return FL_TEST_RAW(obj, OBJ_FIELD_HEAP) || (id2ref_tbl && rb_obj_shape_has_id(obj)); } + UNREACHABLE_RETURN(true); } -void -rb_global_variable(var) - VALUE *var; +/* + * Returns true if the object requires a full rb_gc_obj_free() call during sweep, + * false if it can be freed quickly without calling destructors or cleanup. + * + * Objects that return false are: + * - Simple embedded objects without external allocations + * - Objects without finalizers + * - Objects without object IDs registered in id2ref + * - Objects without generic instance variables + * + * This is used by the GC sweep fast path to avoid function call overhead + * for the majority of simple objects. + */ +bool +rb_gc_obj_needs_cleanup_p(VALUE obj) { - rb_gc_register_address(var); + VALUE flags = RBASIC(obj)->flags; + + if (flags & FL_FINALIZE) return true; + + switch (flags & RUBY_T_MASK) { + case T_IMEMO: + return rb_gc_imemo_needs_cleanup_p(obj); + + case T_DATA: + case T_OBJECT: + case T_STRING: + case T_ARRAY: + case T_HASH: + case T_BIGNUM: + case T_STRUCT: + case T_FLOAT: + case T_RATIONAL: + case T_COMPLEX: + case T_MATCH: + break; + + case T_FILE: + case T_SYMBOL: + case T_CLASS: + case T_ICLASS: + case T_MODULE: + case T_REGEXP: + return true; + } + + shape_id_t shape_id = RBASIC_SHAPE_ID(obj); + if (id2ref_tbl && rb_shape_has_object_id(shape_id)) return true; + + switch (flags & RUBY_T_MASK) { + case T_OBJECT: + if (flags & ROBJECT_HEAP) return true; + return false; + + case T_DATA: + { + uintptr_t type = (uintptr_t)RTYPEDDATA(obj)->type; + if (type & TYPED_DATA_EMBEDDED) { + RUBY_DATA_FUNC dfree = ((const rb_data_type_t *)(type & TYPED_DATA_PTR_MASK))->function.dfree; + if (dfree == RUBY_NEVER_FREE || dfree == RUBY_TYPED_DEFAULT_FREE) { + return false; + } + } + } + return true; + + case T_STRING: + if (flags & (RSTRING_NOEMBED | RSTRING_FSTR)) return true; + return rb_shape_has_fields(shape_id); + + case T_ARRAY: + if (!(flags & RARRAY_EMBED_FLAG)) return true; + return rb_shape_has_fields(shape_id); + + case T_HASH: + if (flags & RHASH_ST_TABLE_FLAG) return true; + return rb_shape_has_fields(shape_id); + + case T_MATCH: + if ((flags & (RMATCH_ONIG | RMATCH_OFFSETS_EXTERNAL)) || USE_DEBUG_COUNTER) return true; + return rb_shape_has_fields(shape_id); + + case T_BIGNUM: + if (!(flags & BIGNUM_EMBED_FLAG)) return true; + return rb_shape_has_fields(shape_id); + + case T_STRUCT: + if (!(flags & RSTRUCT_EMBED_LEN_MASK)) return true; + if (flags & RSTRUCT_GEN_FIELDS) return rb_shape_has_fields(shape_id); + return false; + + case T_FLOAT: + case T_RATIONAL: + case T_COMPLEX: + return rb_shape_has_fields(shape_id); + + default: + UNREACHABLE_RETURN(true); + } +} + +static void +io_fptr_finalize(void *fptr) +{ + rb_io_fptr_finalize((struct rb_io *)fptr); } -typedef struct RVALUE { - union { - struct { - unsigned long flags; /* always 0 for freed obj */ - struct RVALUE *next; - } free; - struct RBasic basic; - struct RObject object; - struct RClass klass; - struct RFloat flonum; - struct RString string; - struct RArray array; - struct RRegexp regexp; - struct RHash hash; - struct RData data; - struct RStruct rstruct; - struct RBignum bignum; - struct RFile file; - struct RNode node; - struct RMatch match; - struct RVarmap varmap; - struct SCOPE scope; - } as; -} RVALUE; - -static RVALUE *freelist = 0; -static RVALUE *deferred_final_list = 0; - -#define HEAPS_INCREMENT 10 -static struct heaps_slot { - RVALUE *slot; - int limit; -} *heaps; -static int heaps_length = 0; -static int heaps_used = 0; - -#define HEAP_MIN_SLOTS 10000 -static int heap_slots = HEAP_MIN_SLOTS; - -#define FREE_MIN 4096 - -static RVALUE *himem, *lomem; +static inline void +make_io_zombie(void *objspace, VALUE obj) +{ + rb_io_t *fptr = RFILE(obj)->fptr; + rb_gc_impl_make_zombie(objspace, obj, io_fptr_finalize, fptr); +} + +static bool +rb_data_free(void *objspace, VALUE obj) +{ + void *data = RTYPEDDATA_GET_DATA(obj); + if (data) { + int free_immediately = false; + void (*dfree)(void *); + + free_immediately = (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0; + dfree = RTYPEDDATA_TYPE(obj)->function.dfree; + + if (dfree) { + if (dfree == RUBY_DEFAULT_FREE) { + if (!RTYPEDDATA_EMBEDDED_P(obj)) { + xfree(data); + RB_DEBUG_COUNTER_INC(obj_data_xfree); + } + } + else if (free_immediately) { + (*dfree)(data); + if (RTYPEDDATA_EMBEDDABLE_P(obj) && !RTYPEDDATA_EMBEDDED_P(obj)) { + xfree(data); + } + + RB_DEBUG_COUNTER_INC(obj_data_imm_free); + } + else { + rb_gc_impl_make_zombie(objspace, obj, dfree, data); + RB_DEBUG_COUNTER_INC(obj_data_zombie); + return FALSE; + } + } + else { + RB_DEBUG_COUNTER_INC(obj_data_empty); + } + } + + return true; +} + +struct classext_foreach_args { + VALUE klass; + rb_objspace_t *objspace; // used for update_* +}; static void -add_heap() -{ - RVALUE *p, *pend; - - if (heaps_used == heaps_length) { - /* Realloc heaps */ - struct heaps_slot *p; - int length; - - heaps_length += HEAPS_INCREMENT; - length = heaps_length*sizeof(struct heaps_slot); - RUBY_CRITICAL( - if (heaps_used > 0) { - p = (struct heaps_slot *)realloc(heaps, length); - if (p) heaps = p; - } - else { - p = heaps = (struct heaps_slot *)malloc(length); - }); - if (p == 0) rb_memerror(); - } - - for (;;) { - RUBY_CRITICAL(p = heaps[heaps_used].slot = (RVALUE*)malloc(sizeof(RVALUE)*heap_slots)); - heaps[heaps_used].limit = heap_slots; - if (p == 0) { - if (heap_slots == HEAP_MIN_SLOTS) { - rb_memerror(); - } - heap_slots = HEAP_MIN_SLOTS; - continue; - } - break; - } - pend = p + heap_slots; - if (lomem == 0 || lomem > p) lomem = p; - if (himem < pend) himem = pend; - heaps_used++; - heap_slots *= 1.8; - - while (p < pend) { - p->as.free.flags = 0; - p->as.free.next = freelist; - freelist = p; - p++; - } -} -#define RANY(o) ((RVALUE*)(o)) +classext_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) +{ + struct classext_foreach_args *args = (struct classext_foreach_args *)arg; + + rb_class_classext_free(args->klass, ext, is_prime); +} + +static void +classext_iclass_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) +{ + struct classext_foreach_args *args = (struct classext_foreach_args *)arg; + + rb_iclass_classext_free(args->klass, ext, is_prime); +} + +bool +rb_gc_obj_free(void *objspace, VALUE obj) +{ + struct classext_foreach_args args; + + RB_DEBUG_COUNTER_INC(obj_free); + + switch (BUILTIN_TYPE(obj)) { + case T_NIL: + case T_FIXNUM: + case T_TRUE: + case T_FALSE: + rb_bug("obj_free() called for broken object"); + break; + default: + break; + } + + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(obj)) { + RB_DEBUG_COUNTER_INC(obj_obj_complex); + st_free_table(ROBJECT_FIELDS_HASH(obj)); + } + else { + SIZED_FREE_N(ROBJECT(obj)->as.heap.fields, ROBJECT_FIELDS_CAPACITY(obj)); + RB_DEBUG_COUNTER_INC(obj_obj_ptr); + } + } + else { + RB_DEBUG_COUNTER_INC(obj_obj_embed); + } + break; + case T_MODULE: + case T_CLASS: +#if USE_ZJIT + rb_zjit_klass_free(obj); +#endif + args.klass = obj; + rb_class_classext_foreach(obj, classext_free, (void *)&args); + if (RCLASS_CLASSEXT_TBL(obj)) { + st_free_table(RCLASS_CLASSEXT_TBL(obj)); + } + (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE); + (void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS); + break; + case T_STRING: + rb_str_free(obj); + break; + case T_ARRAY: + rb_ary_free(obj); + break; + case T_HASH: +#if USE_DEBUG_COUNTER + switch (RHASH_SIZE(obj)) { + case 0: + RB_DEBUG_COUNTER_INC(obj_hash_empty); + break; + case 1: + RB_DEBUG_COUNTER_INC(obj_hash_1); + break; + case 2: + RB_DEBUG_COUNTER_INC(obj_hash_2); + break; + case 3: + RB_DEBUG_COUNTER_INC(obj_hash_3); + break; + case 4: + RB_DEBUG_COUNTER_INC(obj_hash_4); + break; + case 5: + case 6: + case 7: + case 8: + RB_DEBUG_COUNTER_INC(obj_hash_5_8); + break; + default: + GC_ASSERT(RHASH_SIZE(obj) > 8); + RB_DEBUG_COUNTER_INC(obj_hash_g8); + } + + if (RHASH_AR_TABLE_P(obj)) { + if (RHASH_AR_TABLE(obj) == NULL) { + RB_DEBUG_COUNTER_INC(obj_hash_null); + } + else { + RB_DEBUG_COUNTER_INC(obj_hash_ar); + } + } + else { + RB_DEBUG_COUNTER_INC(obj_hash_st); + } +#endif + + rb_hash_free(obj); + break; + case T_REGEXP: + if (RREGEXP(obj)->ptr) { + onig_free(RREGEXP(obj)->ptr); + RB_DEBUG_COUNTER_INC(obj_regexp_ptr); + } + break; + case T_DATA: + if (!rb_data_free(objspace, obj)) return false; + break; + case T_MATCH: + { + struct RMatch *rm = RMATCH(obj); +#if USE_DEBUG_COUNTER + if (rm->num_regs >= 8) { + RB_DEBUG_COUNTER_INC(obj_match_ge8); + } + else if (rm->num_regs >= 4) { + RB_DEBUG_COUNTER_INC(obj_match_ge4); + } + else if (rm->num_regs >= 1) { + RB_DEBUG_COUNTER_INC(obj_match_under4); + } +#endif + if (FL_TEST_RAW(obj, RMATCH_ONIG)) { + onig_region_free(&rm->as.onig, 0); + } + SIZED_FREE_N(rm->char_offset, rm->char_offset_num_allocated); + + RB_DEBUG_COUNTER_INC(obj_match_ptr); + } + break; + case T_FILE: + if (RFILE(obj)->fptr) { + bool closed = rb_io_fptr_finalize_closed(RFILE(obj)->fptr); + if (!closed) make_io_zombie(objspace, obj); + RB_DEBUG_COUNTER_INC(obj_file_ptr); + return closed; + } + break; + case T_RATIONAL: + RB_DEBUG_COUNTER_INC(obj_rational); + break; + case T_COMPLEX: + RB_DEBUG_COUNTER_INC(obj_complex); + break; + case T_MOVED: + break; + case T_ICLASS: + args.klass = obj; + + rb_class_classext_foreach(obj, classext_iclass_free, (void *)&args); + if (RCLASS_CLASSEXT_TBL(obj)) { + st_free_table(RCLASS_CLASSEXT_TBL(obj)); + } + + RB_DEBUG_COUNTER_INC(obj_iclass_ptr); + break; + + case T_FLOAT: + RB_DEBUG_COUNTER_INC(obj_float); + break; + + case T_BIGNUM: + if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) { + SIZED_FREE_N(BIGNUM_DIGITS(obj), BIGNUM_LEN(obj)); + RB_DEBUG_COUNTER_INC(obj_bignum_ptr); + } + else { + RB_DEBUG_COUNTER_INC(obj_bignum_embed); + } + break; + + case T_NODE: + UNEXPECTED_NODE(obj_free); + break; + + case T_STRUCT: + if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) || + RSTRUCT(obj)->as.heap.ptr == NULL) { + RB_DEBUG_COUNTER_INC(obj_struct_embed); + } + else { + SIZED_FREE_N(RSTRUCT(obj)->as.heap.ptr, RSTRUCT(obj)->as.heap.len); + RB_DEBUG_COUNTER_INC(obj_struct_ptr); + } + break; + + case T_SYMBOL: + RB_DEBUG_COUNTER_INC(obj_symbol); + break; + + case T_IMEMO: + rb_imemo_free((VALUE)obj); + break; + + default: + rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE, + BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags); + } + + if (FL_TEST_RAW(obj, FL_FINALIZE)) { + rb_gc_impl_make_zombie(objspace, obj, 0, 0); + return FALSE; + } + else { + return TRUE; + } +} + +void +rb_objspace_set_event_hook(const rb_event_flag_t event) +{ + rb_gc_impl_set_event_hook(rb_gc_get_objspace(), event); +} + +static int +internal_object_p(VALUE obj) +{ + void *ptr = asan_unpoison_object_temporary(obj); + + if (RBASIC(obj)->flags) { + switch (BUILTIN_TYPE(obj)) { + case T_NODE: + UNEXPECTED_NODE(internal_object_p); + break; + case T_NONE: + case T_MOVED: + case T_IMEMO: + case T_ICLASS: + case T_ZOMBIE: + break; + case T_CLASS: + if (obj == rb_mRubyVMFrozenCore) + return 1; + + if (!RBASIC_CLASS(obj)) break; + if (RCLASS_SINGLETON_P(obj)) { + return rb_singleton_class_internal_p(obj); + } + return 0; + default: + if (!RBASIC(obj)->klass) break; + return 0; + } + } + if (ptr || !RBASIC(obj)->flags) { + rb_asan_poison_object(obj); + } + return 1; +} + +int +rb_objspace_internal_object_p(VALUE obj) +{ + return internal_object_p(obj); +} + +struct os_each_struct { + size_t num; + VALUE of; +}; + +static int +os_obj_of_i(void *vstart, void *vend, size_t stride, void *data) +{ + struct os_each_struct *oes = (struct os_each_struct *)data; + + VALUE v = (VALUE)vstart; + for (; v != (VALUE)vend; v += stride) { + if (!internal_object_p(v)) { + if (!oes->of || rb_obj_is_kind_of(v, oes->of)) { + if (!rb_multi_ractor_p() || rb_ractor_shareable_p(v)) { + rb_yield(v); + oes->num++; + } + } + } + } + + return 0; +} + +static VALUE +os_obj_of(VALUE of) +{ + struct os_each_struct oes; + + oes.num = 0; + oes.of = of; + rb_objspace_each_objects(os_obj_of_i, &oes); + return SIZET2NUM(oes.num); +} + +/* + * call-seq: + * ObjectSpace.each_object([module]) {|obj| ... } -> integer + * ObjectSpace.each_object([module]) -> an_enumerator + * + * Calls the block once for each living, nonimmediate object in this + * Ruby process. If <i>module</i> is specified, calls the block + * for only those classes or modules that match (or are a subclass of) + * <i>module</i>. Returns the number of objects found. Immediate + * objects (such as <code>Fixnum</code>s, static <code>Symbol</code>s + * <code>true</code>, <code>false</code> and <code>nil</code>) are + * never returned. + * + * If no block is given, an enumerator is returned instead. + * + * Job = Class.new + * jobs = [Job.new, Job.new] + * count = ObjectSpace.each_object(Job) {|x| p x } + * puts "Total count: #{count}" + * + * <em>produces:</em> + * + * #<Job:0x000000011d6cbbf0> + * #<Job:0x000000011d6cbc68> + * Total count: 2 + * + * Due to a current Ractor implementation issue, this method does not yield + * Ractor-unshareable objects when the process is in multi-Ractor mode. Multi-ractor + * mode is enabled when <code>Ractor.new</code> has been called for the first time. + * See https://bugs.ruby-lang.org/issues/19387 for more information. + * + * a = 12345678987654321 # shareable + * b = [].freeze # shareable + * c = {} # not shareable + * ObjectSpace.each_object {|x| x } # yields a, b, and c + * Ractor.new {} # enter multi-Ractor mode + * ObjectSpace.each_object {|x| x } # does not yield c + * + */ + +static VALUE +os_each_obj(int argc, VALUE *argv, VALUE os) +{ + VALUE of; + + of = (!rb_check_arity(argc, 0, 1) ? 0 : argv[0]); + RETURN_ENUMERATOR(os, 1, &of); + return os_obj_of(of); +} + +/* + * call-seq: + * ObjectSpace.undefine_finalizer(obj) + * + * Removes all finalizers for <i>obj</i>. + * + */ + +static VALUE +undefine_final(VALUE os, VALUE obj) +{ + return rb_undefine_finalizer(obj); +} VALUE -rb_newobj() +rb_undefine_finalizer(VALUE obj) { - VALUE obj; + rb_check_frozen(obj); - if (!freelist) rb_gc(); + rb_gc_impl_undefine_finalizer(rb_gc_get_objspace(), obj); - obj = (VALUE)freelist; - freelist = freelist->as.free.next; - MEMZERO((void*)obj, RVALUE, 1); return obj; } +static void +should_be_callable(VALUE block) +{ + if (!rb_obj_respond_to(block, idCall, TRUE)) { + rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)", + rb_obj_class(block)); + } +} + +static void +should_be_finalizable(VALUE obj) +{ + if (!FL_ABLE(obj)) { + rb_raise(rb_eArgError, "cannot define finalizer for %s", + rb_obj_classname(obj)); + } + rb_check_frozen(obj); +} + +void +rb_gc_copy_finalizer(VALUE dest, VALUE obj) +{ + rb_gc_impl_copy_finalizer(rb_gc_get_objspace(), dest, obj); +} + +/* + * call-seq: + * ObjectSpace.define_finalizer(obj) {|id| ... } -> array + * ObjectSpace.define_finalizer(obj, finalizer) -> array + * + * Adds a new finalizer for +obj+ that is called when +obj+ is destroyed + * by the garbage collector or when Ruby shuts down (which ever comes first). + * + * With a block given, uses the block as the callback. Without a block given, + * uses a callable object +finalizer+ as the callback. The callback is called + * when +obj+ is destroyed with a single argument +id+ which is the object + * ID of +obj+ (see Object#object_id). + * + * The return value is an array <code>[0, callback]</code>, where +callback+ + * is a Proc created from the block if one was given or +finalizer+ otherwise. + * + * Note that defining a finalizer in an instance method of the object may prevent + * the object from being garbage collected since if the block or +finalizer+ refers + * to +obj+ then +obj+ will never be reclaimed by the garbage collector. For example, + * the following script demonstrates the issue: + * + * class Foo + * def define_final + * ObjectSpace.define_finalizer(self) do |id| + * puts "Running finalizer for #{id}!" + * end + * end + * end + * + * obj = Foo.new + * obj.define_final + * + * There are two patterns to solve this issue: + * + * - Create the finalizer in a non-instance method so it can safely capture + * the needed state: + * + * class Foo + * def define_final + * ObjectSpace.define_finalizer(self, self.class.create_finalizer) + * end + * + * def self.create_finalizer + * proc do |id| + * puts "Running finalizer for #{id}!" + * end + * end + * end + * + * - Use a callable object: + * + * class Foo + * class Finalizer + * def call(id) + * puts "Running finalizer for #{id}!" + * end + * end + * + * def define_final + * ObjectSpace.define_finalizer(self, Finalizer.new) + * end + * end + * + * Note that finalization can be unpredictable and is never guaranteed + * to be run except on exit. + */ + +static VALUE +define_final(int argc, VALUE *argv, VALUE os) +{ + VALUE obj, block; + + rb_scan_args(argc, argv, "11", &obj, &block); + if (argc == 1) { + block = rb_block_proc(); + } + + if (rb_callable_receiver(block) == obj) { + rb_warn("finalizer references object to be finalized"); + } + + return rb_define_finalizer(obj, block); +} + VALUE -rb_data_object_alloc(klass, datap, dmark, dfree) - VALUE klass; - void *datap; - RUBY_DATA_FUNC dmark; - RUBY_DATA_FUNC dfree; +rb_define_finalizer(VALUE obj, VALUE block) { - NEWOBJ(data, struct RData); - Check_Type(klass, T_CLASS); - OBJSETUP(data, klass, T_DATA); - data->data = datap; - data->dfree = dfree; - data->dmark = dmark; + should_be_finalizable(obj); + should_be_callable(block); - return (VALUE)data; + block = rb_gc_impl_define_finalizer(rb_gc_get_objspace(), obj, block); + + block = rb_ary_new3(2, INT2FIX(0), block); + OBJ_FREEZE(block); + return block; } -extern st_table *rb_class_tbl; -VALUE *rb_gc_stack_start = 0; +void +rb_objspace_call_finalizer(void) +{ + rb_gc_impl_shutdown_call_finalizer(rb_gc_get_objspace()); +} -#if defined(DJGPP) || defined(_WIN32_WCE) -static unsigned int STACK_LEVEL_MAX = 65535; -#else -#ifdef __human68k__ -extern unsigned int _stacksize; -# define STACK_LEVEL_MAX (_stacksize - 4096) -# undef HAVE_GETRLIMIT +void +rb_objspace_free_objects(void *objspace) +{ + rb_gc_impl_shutdown_free_objects(objspace); +} + +int +rb_objspace_garbage_object_p(VALUE obj) +{ + return !SPECIAL_CONST_P(obj) && rb_gc_impl_garbage_object_p(rb_gc_get_objspace(), obj); +} + +bool +rb_gc_pointer_to_heap_p(VALUE obj) +{ + return rb_gc_impl_pointer_to_heap_p(rb_gc_get_objspace(), (void *)obj); +} + +#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1) +#define LAST_OBJECT_ID() (object_id_counter * OBJ_ID_INCREMENT) +static VALUE id2ref_value = 0; + +#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG +static size_t object_id_counter = 1; #else -#ifdef HAVE_GETRLIMIT -static unsigned int STACK_LEVEL_MAX = 655300; +static unsigned long long object_id_counter = 1; +#endif + +static inline VALUE +generate_next_object_id(void) +{ +#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG + // 64bit atomics are available + return SIZET2NUM(RUBY_ATOMIC_SIZE_FETCH_ADD(object_id_counter, 1) * OBJ_ID_INCREMENT); #else -# define STACK_LEVEL_MAX 655300 + unsigned int lock_lev = RB_GC_VM_LOCK(); + VALUE id = ULL2NUM(++object_id_counter * OBJ_ID_INCREMENT); + RB_GC_VM_UNLOCK(lock_lev); + return id; #endif +} + +void +rb_gc_obj_id_moved(VALUE obj) +{ + if (UNLIKELY(id2ref_tbl)) { + st_insert(id2ref_tbl, (st_data_t)rb_obj_id(obj), (st_data_t)obj); + } +} + +static int +object_id_cmp(st_data_t x, st_data_t y) +{ + if (RB_TYPE_P(x, T_BIGNUM)) { + return !rb_big_eql(x, y); + } + else { + return x != y; + } +} + +static st_index_t +object_id_hash(st_data_t n) +{ + return FIX2LONG(rb_hash((VALUE)n)); +} + +static const struct st_hash_type object_id_hash_type = { + object_id_cmp, + object_id_hash, +}; + +static void gc_mark_tbl_no_pin(st_table *table); + +static void +id2ref_tbl_mark(void *data) +{ + st_table *table = (st_table *)data; + if (UNLIKELY(!RB_POSFIXABLE(LAST_OBJECT_ID()))) { + // It's very unlikely, but if enough object ids were generated, keys may be T_BIGNUM + rb_mark_set(table); + } + // We purposely don't mark values, as they are weak references. + // rb_gc_obj_free_vm_weak_references takes care of cleaning them up. +} + +static size_t +id2ref_tbl_memsize(const void *data) +{ + return rb_st_memsize(data); +} + +static void +id2ref_tbl_free(void *data) +{ + id2ref_tbl = NULL; // clear global ref + st_table *table = (st_table *)data; + st_free_table(table); +} + +static const rb_data_type_t id2ref_tbl_type = { + .wrap_struct_name = "VM/_id2ref_table", + .function = { + .dmark = id2ref_tbl_mark, + .dfree = id2ref_tbl_free, + .dsize = id2ref_tbl_memsize, + // dcompact function not required because the table is reference updated + // in rb_gc_vm_weak_table_foreach + }, + .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY +}; + +static VALUE +class_object_id(VALUE klass) +{ + VALUE id = RUBY_ATOMIC_VALUE_LOAD(RCLASS(klass)->object_id); + if (!id) { + unsigned int lock_lev = RB_GC_VM_LOCK(); + id = generate_next_object_id(); + VALUE existing_id = RUBY_ATOMIC_VALUE_CAS(RCLASS(klass)->object_id, 0, id); + if (existing_id) { + id = existing_id; + } + else if (RB_UNLIKELY(id2ref_tbl)) { + st_insert(id2ref_tbl, id, klass); + } + RB_GC_VM_UNLOCK(lock_lev); + } + return id; +} + +static inline VALUE +object_id_get(VALUE obj, shape_id_t shape_id) +{ + VALUE id; + if (rb_shape_complex_p(shape_id)) { + id = rb_obj_field_get(obj, ROOT_COMPLEX_WITH_OBJ_ID); + } + else { + id = rb_obj_field_get(obj, rb_shape_object_id(shape_id)); + } + +#if RUBY_DEBUG + if (!(FIXNUM_P(id) || RB_TYPE_P(id, T_BIGNUM))) { + rb_p(obj); + rb_bug("Object's shape includes object_id, but it's missing %s", rb_obj_info(obj)); + } #endif + + return id; +} + +static VALUE +object_id0(VALUE obj) +{ + VALUE id = Qfalse; + shape_id_t shape_id = RBASIC_SHAPE_ID(obj); + + if (rb_shape_has_object_id(shape_id)) { + return object_id_get(obj, shape_id); + } + + shape_id_t object_id_shape_id = rb_obj_shape_transition_object_id(obj); + + id = generate_next_object_id(); + rb_obj_field_set(obj, object_id_shape_id, 0, id); + + RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == object_id_shape_id); + RUBY_ASSERT(rb_obj_shape_has_id(obj)); + + if (RB_UNLIKELY(id2ref_tbl)) { + RB_VM_LOCKING() { + st_insert(id2ref_tbl, (st_data_t)id, (st_data_t)obj); + } + } + return id; +} + +static VALUE +object_id(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_CLASS: + case T_MODULE: + // With Ruby Box, classes and modules have different fields + // in different boxes, so we cannot store the object id + // in fields. + return class_object_id(obj); + case T_IMEMO: + RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); + break; + default: + break; + } + + if (UNLIKELY(rb_gc_multi_ractor_p() && rb_ractor_shareable_p(obj))) { + unsigned int lock_lev = RB_GC_VM_LOCK(); + VALUE id = object_id0(obj); + RB_GC_VM_UNLOCK(lock_lev); + return id; + } + + return object_id0(obj); +} + +static void +build_id2ref_i(VALUE obj, void *data) +{ + st_table *id2ref_tbl = (st_table *)data; + + switch (BUILTIN_TYPE(obj)) { + case T_CLASS: + case T_MODULE: + RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); + if (RCLASS(obj)->object_id) { + st_insert(id2ref_tbl, RCLASS(obj)->object_id, obj); + } + break; + case T_IMEMO: + RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); + if (IMEMO_TYPE_P(obj, imemo_fields) && rb_obj_shape_has_id(obj)) { + st_insert(id2ref_tbl, rb_obj_id(obj), rb_imemo_fields_owner(obj)); + } + break; + case T_OBJECT: + RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); + if (rb_obj_shape_has_id(obj)) { + st_insert(id2ref_tbl, rb_obj_id(obj), obj); + } + break; + default: + // For generic_fields, the T_IMEMO/fields is responsible for populating the entry. + break; + } +} + +static VALUE +object_id_to_ref(void *objspace_ptr, VALUE object_id) +{ + rb_objspace_t *objspace = objspace_ptr; + + unsigned int lev = RB_GC_VM_LOCK(); + + if (!id2ref_tbl) { + rb_gc_vm_barrier(); // stop other ractors + + // GC Must not trigger while we build the table, otherwise if we end + // up freeing an object that had an ID, we might try to delete it from + // the table even though it wasn't inserted yet. + st_table *tmp_id2ref_tbl = st_init_table(&object_id_hash_type); + VALUE tmp_id2ref_value = TypedData_Wrap_Struct(0, &id2ref_tbl_type, tmp_id2ref_tbl); + + // build_id2ref_i will most certainly malloc, which could trigger GC and sweep + // objects we just added to the table. + // By calling rb_gc_disable() we also save having to handle potentially garbage objects. + bool gc_disabled = RTEST(rb_gc_disable()); + { + id2ref_tbl = tmp_id2ref_tbl; + id2ref_value = tmp_id2ref_value; + + rb_gc_impl_each_object(objspace, build_id2ref_i, (void *)id2ref_tbl); + } + if (!gc_disabled) rb_gc_enable(); + } + + VALUE obj; + bool found = st_lookup(id2ref_tbl, object_id, &obj) && !rb_gc_impl_garbage_object_p(objspace, obj); + + RB_GC_VM_UNLOCK(lev); + + if (found) { + return obj; + } + + if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(LAST_OBJECT_ID()))) { + rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not an id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); + } + else { + rb_raise(rb_eRangeError, "%+"PRIsVALUE" is a recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); + } +} + +static inline void +obj_free_object_id(VALUE obj) +{ + VALUE obj_id = 0; + if (RB_UNLIKELY(id2ref_tbl)) { + switch (BUILTIN_TYPE(obj)) { + case T_CLASS: + case T_MODULE: + obj_id = RCLASS(obj)->object_id; + break; + case T_IMEMO: + if (!IMEMO_TYPE_P(obj, imemo_fields)) { + return; + } + // fallthrough + case T_OBJECT: + { + shape_id_t shape_id = RBASIC_SHAPE_ID(obj); + if (rb_shape_has_object_id(shape_id)) { + obj_id = object_id_get(obj, shape_id); + } + break; + } + default: + // For generic_fields, the T_IMEMO/fields is responsible for freeing the id. + return; + } + + if (RB_UNLIKELY(obj_id)) { + RUBY_ASSERT(FIXNUM_P(obj_id) || RB_TYPE_P(obj_id, T_BIGNUM)); + + if (!st_delete(id2ref_tbl, (st_data_t *)&obj_id, NULL)) { + // The the object is a T_IMEMO/fields, then it's possible the actual object + // has been garbage collected already. + if (!RB_TYPE_P(obj, T_IMEMO)) { + rb_bug("Object ID seen, but not in _id2ref table: object_id=%llu object=%s", NUM2ULL(obj_id), rb_obj_info(obj)); + } + } + } + } +} + +void +rb_gc_obj_free_vm_weak_references(VALUE obj) +{ + ASSUME(!RB_SPECIAL_CONST_P(obj)); + obj_free_object_id(obj); + + if (rb_obj_gen_fields_p(obj)) { + rb_free_generic_ivar(obj); + } + + switch (BUILTIN_TYPE(obj)) { + case T_STRING: + if (FL_TEST_RAW(obj, RSTRING_FSTR)) { + rb_gc_free_fstring(obj); + } + break; + case T_SYMBOL: + rb_gc_free_dsymbol(obj); + break; + case T_IMEMO: + switch (imemo_type(obj)) { + case imemo_callinfo: + rb_vm_ci_free((const struct rb_callinfo *)obj); + break; + case imemo_ment: + rb_free_method_entry_vm_weak_references((const rb_method_entry_t *)obj); + break; + default: + break; + } + break; + default: + break; + } +} + +/* + * call-seq: + * ObjectSpace._id2ref(object_id) -> an_object + * + * Converts an object id to a reference to the object. May not be + * called on an object id passed as a parameter to a finalizer. + * + * s = "I am a string" #=> "I am a string" + * r = ObjectSpace._id2ref(s.object_id) #=> "I am a string" + * r == s #=> true + * + * On multi-ractor mode, if the object is not shareable, it raises + * RangeError. + * + * This method is deprecated and should no longer be used. + */ + +static VALUE +id2ref(VALUE objid) +{ + objid = rb_to_int(objid); + if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) { + VALUE ptr = (VALUE)NUM2PTR(objid); + if (SPECIAL_CONST_P(ptr)) { + if (ptr == Qtrue) return Qtrue; + if (ptr == Qfalse) return Qfalse; + if (NIL_P(ptr)) return Qnil; + if (FIXNUM_P(ptr)) return ptr; + if (FLONUM_P(ptr)) return ptr; + + if (SYMBOL_P(ptr)) { + // Check that the symbol is valid + if (rb_static_id_valid_p(SYM2ID(ptr))) { + return ptr; + } + else { + rb_raise(rb_eRangeError, "%p is not a symbol id value", (void *)ptr); + } + } + + rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not an id value", rb_int2str(objid, 10)); + } + } + + VALUE obj = object_id_to_ref(rb_gc_get_objspace(), objid); + if (!rb_multi_ractor_p() || rb_ractor_shareable_p(obj)) { + return obj; + } + else { + rb_raise(rb_eRangeError, "%+"PRIsVALUE" is the id of an unshareable object on multi-ractor", rb_int2str(objid, 10)); + } +} + +/* :nodoc: */ +static VALUE +os_id2ref(VALUE os, VALUE objid) +{ + rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "ObjectSpace._id2ref is deprecated"); + return id2ref(objid); +} + +static VALUE +rb_find_object_id(void *objspace, VALUE obj, VALUE (*get_heap_object_id)(VALUE)) +{ + if (SPECIAL_CONST_P(obj)) { +#if SIZEOF_LONG == SIZEOF_VOIDP + return LONG2NUM((SIGNED_VALUE)obj); +#else + return LL2NUM((SIGNED_VALUE)obj); #endif + } + + return get_heap_object_id(obj); +} -#ifdef C_ALLOCA -# define SET_STACK_END VALUE stack_end; alloca(0); -# define STACK_END (&stack_end) +static VALUE +nonspecial_obj_id(VALUE obj) +{ +#if SIZEOF_LONG == SIZEOF_VOIDP + return (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG); +#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP + return LL2NUM((SIGNED_VALUE)(obj) / 2); #else -# if defined(__GNUC__) && defined(USE_BUILTIN_FRAME_ADDRESS) && !defined(__ia64__) -# define SET_STACK_END VALUE *stack_end = __builtin_frame_address(0) -# else -# define SET_STACK_END VALUE *stack_end = alloca(1) -# endif -# define STACK_END (stack_end) +# error not supported #endif -#if defined(sparc) || defined(__sparc__) -# define STACK_LENGTH (rb_gc_stack_start - STACK_END + 0x80) -#elif STACK_GROW_DIRECTION < 0 -# define STACK_LENGTH (rb_gc_stack_start - STACK_END) +} + +VALUE +rb_memory_id(VALUE obj) +{ + return rb_find_object_id(NULL, obj, nonspecial_obj_id); +} + +/* + * Document-method: __id__ + * Document-method: object_id + * + * call-seq: + * obj.__id__ -> integer + * obj.object_id -> integer + * + * Returns an integer identifier for +obj+. + * + * The same number will be returned on all calls to +object_id+ for a given + * object, and no two active objects will share an id. + * + * Note: that some objects of builtin classes are reused for optimization. + * This is the case for immediate values and frozen string literals. + * + * BasicObject implements +__id__+, Kernel implements +object_id+. + * + * Immediate values are not passed by reference but are passed by value: + * +nil+, +true+, +false+, Fixnums, Symbols, and some Floats. + * + * Object.new.object_id == Object.new.object_id # => false + * (21 * 2).object_id == (21 * 2).object_id # => true + * "hello".object_id == "hello".object_id # => false + * "hi".freeze.object_id == "hi".freeze.object_id # => true + */ + +VALUE +rb_obj_id(VALUE obj) +{ + /* If obj is an immediate, the object ID is obj directly converted to a Numeric. + * Otherwise, the object ID is a Numeric that is a non-zero multiple of + * (RUBY_IMMEDIATE_MASK + 1) which guarantees that it does not collide with + * any immediates. */ + return rb_find_object_id(rb_gc_get_objspace(), obj, object_id); +} + +bool +rb_obj_id_p(VALUE obj) +{ + return !RB_TYPE_P(obj, T_IMEMO) && rb_obj_shape_has_id(obj); +} + +/* + * GC implementations should call this function before the GC phase that updates references + * embedded in the machine code generated by JIT compilers. JIT compilers usually enforce the + * "W^X" policy and protect the code memory from being modified during execution. This function + * makes the code memory writeable. + */ +void +rb_gc_before_updating_jit_code(void) +{ +#if USE_YJIT + rb_yjit_mark_all_writeable(); +#endif +#if USE_ZJIT + rb_zjit_mark_all_writable(); +#endif +} + +/* + * GC implementations should call this function before the GC phase that updates references + * embedded in the machine code generated by JIT compilers. This function makes the code memory + * executable again. + */ +void +rb_gc_after_updating_jit_code(void) +{ +#if USE_YJIT + rb_yjit_mark_all_executable(); +#endif +#if USE_ZJIT + rb_zjit_mark_all_executable(); +#endif +} + +static void +classext_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) +{ + size_t *size = (size_t *)arg; + size_t s = 0; + + if (RCLASSEXT_M_TBL(ext)) { + s += rb_id_table_memsize(RCLASSEXT_M_TBL(ext)); + } + if (RCLASSEXT_CONST_TBL(ext)) { + s += rb_id_table_memsize(RCLASSEXT_CONST_TBL(ext)); + } + if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) { + s += (RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1) * sizeof(VALUE); + } + if (!prime) { + s += sizeof(rb_classext_t); + } + *size += s; +} + +static void +classext_superclasses_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) +{ + size_t *size = (size_t *)arg; + size_t array_size; + if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) { + RUBY_ASSERT(prime); + array_size = RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1; + *size += array_size * sizeof(VALUE); + } +} + +size_t +rb_obj_memsize_of(VALUE obj) +{ + size_t size = 0; + + if (SPECIAL_CONST_P(obj)) { + return 0; + } + + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(obj)) { + size += rb_st_memsize(ROBJECT_FIELDS_HASH(obj)); + } + else { + size += ROBJECT_FIELDS_CAPACITY(obj) * sizeof(VALUE); + } + } + break; + case T_MODULE: + case T_CLASS: + rb_class_classext_foreach(obj, classext_memsize, (void *)&size); + rb_class_classext_foreach(obj, classext_superclasses_memsize, (void *)&size); + break; + case T_ICLASS: + if (RICLASS_OWNS_M_TBL_P(obj)) { + if (RCLASS_M_TBL(obj)) { + size += rb_id_table_memsize(RCLASS_M_TBL(obj)); + } + } + break; + case T_STRING: + size += rb_str_memsize(obj); + break; + case T_ARRAY: + size += rb_ary_memsize(obj); + break; + case T_HASH: + if (RHASH_ST_TABLE_P(obj)) { + VM_ASSERT(RHASH_ST_TABLE(obj) != NULL); + /* st_table is in the slot */ + size += st_memsize(RHASH_ST_TABLE(obj)) - sizeof(st_table); + } + break; + case T_REGEXP: + if (RREGEXP_PTR(obj)) { + size += onig_memsize(RREGEXP_PTR(obj)); + } + break; + case T_DATA: + size += rb_objspace_data_type_memsize(obj); + break; + case T_MATCH: + { + struct RMatch *rm = RMATCH(obj); + if (FL_TEST_RAW(obj, RMATCH_ONIG)) { + size += onig_region_memsize(&rm->as.onig); + } + size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated; + } + break; + case T_FILE: + if (RFILE(obj)->fptr) { + size += rb_io_memsize(RFILE(obj)->fptr); + } + break; + case T_RATIONAL: + case T_COMPLEX: + break; + case T_IMEMO: + size += rb_imemo_memsize(obj); + break; + + case T_FLOAT: + case T_SYMBOL: + break; + + case T_BIGNUM: + if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) { + size += BIGNUM_LEN(obj) * sizeof(BDIGIT); + } + break; + + case T_NODE: + UNEXPECTED_NODE(obj_memsize_of); + break; + + case T_STRUCT: + if (RSTRUCT_EMBED_LEN(obj) == 0) { + size += sizeof(VALUE) * RSTRUCT_LEN_RAW(obj); + } + break; + + case T_ZOMBIE: + case T_MOVED: + break; + + default: + rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)", + BUILTIN_TYPE(obj), (void*)obj); + } + + return size + rb_gc_obj_slot_size(obj); +} + +static int +set_zero(st_data_t key, st_data_t val, st_data_t arg) +{ + VALUE k = (VALUE)key; + VALUE hash = (VALUE)arg; + rb_hash_aset(hash, k, INT2FIX(0)); + return ST_CONTINUE; +} + +struct count_objects_data { + size_t counts[T_MASK+1]; + size_t freed; + size_t total; +}; + +static void +count_objects_i(VALUE obj, void *d) +{ + struct count_objects_data *data = (struct count_objects_data *)d; + + if (RBASIC(obj)->flags) { + data->counts[BUILTIN_TYPE(obj)]++; + } + else { + data->freed++; + } + + data->total++; +} + +/* + * call-seq: + * ObjectSpace.count_objects(result_hash = {}) -> hash + * + * Counts the number of objects, grouped by type. + * + * It returns a hash that looks like: + * + * { + * TOTAL: 10000, + * FREE: 3011, + * T_OBJECT: 6, + * T_CLASS: 404, + * # ... + * } + * + * The contents of the returned hash are implementation specific and + * may be changed in future versions without notice. + * + * The keys starting with +:T_+ are live objects of a particular type. + * For example, +:T_ARRAY+ is the number of arrays. + * + * The key +:FREE+ is the number of object slots which are empty. + * + * The key +:TOTAL+ is the total number of slots (which is the sum of + * all of the other values). + * + * If the optional argument +result_hash+ is given, + * it is overwritten and returned. + * This is intended to avoid the probe effect. + * + * h = {} + * ObjectSpace.count_objects(h) + * puts h + * # => { TOTAL: 10000, T_CLASS: 158280, T_MODULE: 20672, T_STRING: 527249 } + * + * This method is only expected to work on C Ruby. + * + */ + +static VALUE +count_objects(int argc, VALUE *argv, VALUE os) +{ + struct count_objects_data data = { 0 }; + VALUE hash = Qnil; + VALUE types[T_MASK + 1]; + + if (rb_check_arity(argc, 0, 1) == 1) { + hash = argv[0]; + if (!RB_TYPE_P(hash, T_HASH)) + rb_raise(rb_eTypeError, "non-hash given"); + } + + for (size_t i = 0; i <= T_MASK; i++) { + // type_sym can allocate an object, + // so we need to create all key symbols in advance + // not to disturb the result + types[i] = type_sym(i); + } + + // Same as type_sym, we need to create all key symbols in advance + VALUE total = ID2SYM(rb_intern("TOTAL")); + VALUE free = ID2SYM(rb_intern("FREE")); + + rb_gc_impl_each_object(rb_gc_get_objspace(), count_objects_i, &data); + + if (NIL_P(hash)) { + hash = rb_hash_new(); + } + else if (!RHASH_EMPTY_P(hash)) { + rb_hash_stlike_foreach(hash, set_zero, hash); + } + rb_hash_aset(hash, total, SIZET2NUM(data.total)); + rb_hash_aset(hash, free, SIZET2NUM(data.freed)); + + for (size_t i = 0; i <= T_MASK; i++) { + if (data.counts[i]) { + rb_hash_aset(hash, types[i], SIZET2NUM(data.counts[i])); + } + } + + return hash; +} + +#define SET_STACK_END SET_MACHINE_STACK_END(&ec->machine.stack_end) + +#define STACK_START (ec->machine.stack_start) +#define STACK_END (ec->machine.stack_end) +#define STACK_LEVEL_MAX (ec->machine.stack_maxsize/sizeof(VALUE)) + +#if STACK_GROW_DIRECTION < 0 +# define STACK_LENGTH (size_t)(STACK_START - STACK_END) #elif STACK_GROW_DIRECTION > 0 -# define STACK_LENGTH (STACK_END - rb_gc_stack_start + 1) +# define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1) #else -# define STACK_LENGTH ((STACK_END < rb_gc_stack_start) ? rb_gc_stack_start - STACK_END\ - : STACK_END - rb_gc_stack_start + 1) +# define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \ + : (size_t)(STACK_END - STACK_START + 1)) #endif -#if STACK_GROW_DIRECTION > 0 -# define STACK_UPPER(x, a, b) a -#elif STACK_GROW_DIRECTION < 0 -# define STACK_UPPER(x, a, b) b +#if !STACK_GROW_DIRECTION +int ruby_stack_grow_direction; +int +ruby_get_stack_grow_direction(volatile VALUE *addr) +{ + VALUE *end; + SET_MACHINE_STACK_END(&end); + + if (end > addr) return ruby_stack_grow_direction = 1; + return ruby_stack_grow_direction = -1; +} +#endif + +size_t +ruby_stack_length(VALUE **p) +{ + rb_execution_context_t *ec = GET_EC(); + SET_STACK_END; + if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END); + return STACK_LENGTH; +} + +#define PREVENT_STACK_OVERFLOW 1 +#ifndef PREVENT_STACK_OVERFLOW +#if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK)) +# define PREVENT_STACK_OVERFLOW 1 #else +# define PREVENT_STACK_OVERFLOW 0 +#endif +#endif +#if PREVENT_STACK_OVERFLOW && !defined(__EMSCRIPTEN__) static int -stack_growup_p(addr) - VALUE *addr; +stack_check(rb_execution_context_t *ec, int water_mark) { SET_STACK_END; - if (STACK_END > addr) return Qtrue; - return Qfalse; + size_t length = STACK_LENGTH; + size_t maximum_length = STACK_LEVEL_MAX - water_mark; + + return length > maximum_length; } -# define STACK_UPPER(x, a, b) (stack_growup_p(x) ? a : b) +#else +#define stack_check(ec, water_mark) FALSE #endif -#define GC_WATER_MARK 512 - -#define CHECK_STACK(ret) do {\ - SET_STACK_END;\ - (ret) = (STACK_LENGTH > STACK_LEVEL_MAX + GC_WATER_MARK);\ -} while (0) +#define STACKFRAME_FOR_CALL_CFUNC 2048 int -ruby_stack_length(p) - VALUE **p; +rb_ec_stack_check(rb_execution_context_t *ec) { - SET_STACK_END; - if (p) *p = STACK_UPPER(STACK_END, rb_gc_stack_start, STACK_END); - return STACK_LENGTH; + return stack_check(ec, STACKFRAME_FOR_CALL_CFUNC); } int -ruby_stack_check() +ruby_stack_check(void) { - int ret; + return stack_check(GET_EC(), STACKFRAME_FOR_CALL_CFUNC); +} - CHECK_STACK(ret); - return ret; +/* ==================== Marking ==================== */ + +#define RB_GC_MARK_OR_TRAVERSE(func, obj_or_ptr, obj, check_obj) do { \ + if (!RB_SPECIAL_CONST_P(obj)) { \ + rb_vm_t *vm = GET_VM(); \ + void *objspace = vm->gc.objspace; \ + if (LIKELY(vm->gc.mark_func_data == NULL)) { \ + GC_ASSERT(rb_gc_impl_during_gc_p(objspace)); \ + (func)(objspace, (obj_or_ptr)); \ + } \ + else if (check_obj ? \ + rb_gc_impl_pointer_to_heap_p(objspace, (const void *)obj) && \ + !rb_gc_impl_garbage_object_p(objspace, obj) : \ + true) { \ + GC_ASSERT(!rb_gc_impl_during_gc_p(objspace)); \ + struct gc_mark_func_data_struct *mark_func_data = vm->gc.mark_func_data; \ + vm->gc.mark_func_data = NULL; \ + mark_func_data->mark_func((obj), mark_func_data->data); \ + vm->gc.mark_func_data = mark_func_data; \ + } \ + } \ +} while (0) + +static inline void +gc_mark_internal(VALUE obj) +{ + RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark, obj, obj, false); } -#define MARK_STACK_MAX 1024 -static VALUE mark_stack[MARK_STACK_MAX]; -static VALUE *mark_stack_ptr; -static int mark_stack_overflow; +void +rb_gc_mark_movable(VALUE obj) +{ + gc_mark_internal(obj); +} -static void -init_mark_stack() +void +rb_gc_mark_and_move(VALUE *ptr) { - mark_stack_overflow = 0; - mark_stack_ptr = mark_stack; + RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark_and_move, ptr, *ptr, false); } -#define MARK_STACK_EMPTY (mark_stack_ptr == mark_stack) - -static st_table *source_filenames; +static inline void +gc_mark_and_pin_internal(VALUE obj) +{ + RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark_and_pin, obj, obj, false); +} + +void +rb_gc_mark(VALUE obj) +{ + gc_mark_and_pin_internal(obj); +} -char * -rb_source_filename(f) - const char *f; +static inline void +gc_mark_maybe_internal(VALUE obj) { - char *name; + RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark_maybe, obj, obj, true); +} - if (!st_lookup(source_filenames, (st_data_t)f, (st_data_t *)&name)) { - long len = strlen(f) + 1; - char *ptr = name = ALLOC_N(char, len + 1); - *ptr++ = 0; - MEMCPY(ptr, f, char, len); - st_add_direct(source_filenames, (st_data_t)ptr, (st_data_t)name); - return ptr; +void +rb_gc_mark_maybe(VALUE obj) +{ + gc_mark_maybe_internal(obj); +} + +ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void each_location(register const VALUE *x, register long n, void (*cb)(VALUE, void *), void *data)); +static void +each_location(register const VALUE *x, register long n, void (*cb)(VALUE, void *), void *data) +{ + VALUE v; + while (n--) { + v = *x; + cb(v, data); + x++; } - return name + 1; } static void -mark_source_filename(f) - char *f; +each_location_ptr(const VALUE *start, const VALUE *end, void (*cb)(VALUE, void *), void *data) +{ + if (end <= start) return; + each_location(start, end - start, cb, data); +} + +static void +gc_mark_maybe_each_location(VALUE obj, void *data) +{ + gc_mark_maybe_internal(obj); +} + +void +rb_gc_mark_locations(const VALUE *start, const VALUE *end) +{ + each_location_ptr(start, end, gc_mark_maybe_each_location, NULL); +} + +void +rb_gc_mark_values(long n, const VALUE *values) +{ + for (long i = 0; i < n; i++) { + gc_mark_internal(values[i]); + } +} + +void +rb_gc_mark_vm_stack_values(long n, const VALUE *values) { - if (f) { - f[-1] = 1; + for (long i = 0; i < n; i++) { + gc_mark_and_pin_internal(values[i]); } } static int -sweep_source_filename(key, value) - char *key, *value; +mark_key(st_data_t key, st_data_t value, st_data_t data) +{ + gc_mark_and_pin_internal((VALUE)key); + + return ST_CONTINUE; +} + +void +rb_mark_set(st_table *tbl) +{ + if (!tbl) return; + + st_foreach(tbl, mark_key, (st_data_t)rb_gc_get_objspace()); +} + +static int +mark_keyvalue(st_data_t key, st_data_t value, st_data_t data) +{ + gc_mark_internal((VALUE)key); + gc_mark_internal((VALUE)value); + + return ST_CONTINUE; +} + +static int +pin_key_pin_value(st_data_t key, st_data_t value, st_data_t data) { - if (*value) { - *value = 0; - return ST_CONTINUE; + gc_mark_and_pin_internal((VALUE)key); + gc_mark_and_pin_internal((VALUE)value); + + return ST_CONTINUE; +} + +static int +pin_key_mark_value(st_data_t key, st_data_t value, st_data_t data) +{ + gc_mark_and_pin_internal((VALUE)key); + gc_mark_internal((VALUE)value); + + return ST_CONTINUE; +} + +static void +mark_hash(VALUE hash) +{ + if (rb_hash_compare_by_id_p(hash)) { + rb_hash_stlike_foreach(hash, pin_key_mark_value, 0); } else { - free(value); - return ST_DELETE; + rb_hash_stlike_foreach(hash, mark_keyvalue, 0); } + + gc_mark_internal(RHASH(hash)->ifnone); } -static void gc_mark _((VALUE ptr, int lev)); -static void gc_mark_children _((VALUE ptr, int lev)); +void +rb_mark_hash(st_table *tbl) +{ + if (!tbl) return; + + st_foreach(tbl, pin_key_pin_value, 0); +} + +static enum rb_id_table_iterator_result +mark_method_entry_i(VALUE me, void *objspace) +{ + gc_mark_internal(me); + + return ID_TABLE_CONTINUE; +} static void -gc_mark_all() +mark_m_tbl(void *objspace, struct rb_id_table *tbl) +{ + if (tbl) { + rb_id_table_foreach_values(tbl, mark_method_entry_i, objspace); + } +} + +static enum rb_id_table_iterator_result +mark_const_entry_i(VALUE value, void *objspace) { - RVALUE *p, *pend; - int i; + const rb_const_entry_t *ce = (const rb_const_entry_t *)value; - init_mark_stack(); - for (i = 0; i < heaps_used; i++) { - p = heaps[i].slot; pend = p + heaps[i].limit; - while (p < pend) { - if ((p->as.basic.flags & FL_MARK) && - (p->as.basic.flags != FL_MARK)) { - gc_mark_children((VALUE)p, 0); - } - p++; - } + if (!rb_gc_checking_shareable()) { + gc_mark_internal(ce->value); + gc_mark_internal(ce->file); // TODO: ce->file should be shareable? } + return ID_TABLE_CONTINUE; } static void -gc_mark_rest() +mark_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl) { - VALUE tmp_arry[MARK_STACK_MAX]; - VALUE *p; + if (!tbl) return; + rb_id_table_foreach_values(tbl, mark_const_entry_i, objspace); +} - p = (mark_stack_ptr - mark_stack) + tmp_arry; - MEMCPY(tmp_arry, mark_stack, VALUE, MARK_STACK_MAX); +#if STACK_GROW_DIRECTION < 0 +#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START) +#elif STACK_GROW_DIRECTION > 0 +#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix)) +#else +#define GET_STACK_BOUNDS(start, end, appendix) \ + ((STACK_END < STACK_START) ? \ + ((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix))) +#endif - init_mark_stack(); - while(p != tmp_arry){ - p--; - gc_mark_children(*p, 0); +static void +gc_mark_machine_stack_location_maybe(VALUE obj, void *data) +{ + gc_mark_maybe_internal(obj); + +#ifdef RUBY_ASAN_ENABLED + const rb_execution_context_t *ec = (const rb_execution_context_t *)data; + void *fake_frame_start; + void *fake_frame_end; + bool is_fake_frame = asan_get_fake_stack_extents( + ec->machine.asan_fake_stack_handle, obj, + ec->machine.stack_start, ec->machine.stack_end, + &fake_frame_start, &fake_frame_end + ); + if (is_fake_frame) { + each_location_ptr(fake_frame_start, fake_frame_end, gc_mark_maybe_each_location, NULL); } +#endif } -static inline int -is_pointer_to_heap(ptr) - void *ptr; +static bool +gc_object_moved_p_internal(void *objspace, VALUE obj) { - register RVALUE *p = RANY(ptr); - register RVALUE *heap_org; - register long i; + if (SPECIAL_CONST_P(obj)) { + return false; + } - if (p < lomem || p > himem) return Qfalse; + return rb_gc_impl_object_moved_p(objspace, obj); +} - /* check if p looks like a pointer */ - for (i=0; i < heaps_used; i++) { - heap_org = heaps[i].slot; - if (heap_org <= p && p < heap_org + heaps[i].limit && - ((((char*)p)-((char*)heap_org))%sizeof(RVALUE)) == 0) - return Qtrue; +static VALUE +gc_location_internal(void *objspace, VALUE value) +{ + if (SPECIAL_CONST_P(value)) { + return value; } - return Qfalse; + + GC_ASSERT(rb_gc_impl_pointer_to_heap_p(objspace, (void *)value)); + + return rb_gc_impl_location(objspace, value); +} + +VALUE +rb_gc_location(VALUE value) +{ + return gc_location_internal(rb_gc_get_objspace(), value); } +#if defined(__wasm__) + + +static VALUE *rb_stack_range_tmp[2]; + static void -mark_locations_array(x, n) - register VALUE *x; - register long n; +rb_mark_locations(void *begin, void *end) { - while (n--) { - if (is_pointer_to_heap((void *)*x)) { - gc_mark(*x, 0); - } - x++; - } + rb_stack_range_tmp[0] = begin; + rb_stack_range_tmp[1] = end; } void -rb_gc_mark_locations(start, end) - VALUE *start, *end; +rb_gc_save_machine_context(void) +{ + // no-op +} + +# if defined(__EMSCRIPTEN__) + +static void +mark_current_machine_context(const rb_execution_context_t *ec) { - long n; + emscripten_scan_stack(rb_mark_locations); + each_location_ptr(rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL); - n = end - start; - mark_locations_array(start,n); + emscripten_scan_registers(rb_mark_locations); + each_location_ptr(rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL); } +# else // use Asyncify version -static int -mark_entry(key, value, lev) - ID key; - VALUE value; - int lev; +static void +mark_current_machine_context(rb_execution_context_t *ec) { - gc_mark(value, lev); - return ST_CONTINUE; + VALUE *stack_start, *stack_end; + SET_STACK_END; + GET_STACK_BOUNDS(stack_start, stack_end, 1); + each_location_ptr(stack_start, stack_end, gc_mark_maybe_each_location, NULL); + + rb_wasm_scan_locals(rb_mark_locations); + each_location_ptr(rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL); } +# endif + +#else // !defined(__wasm__) + void -mark_tbl(tbl, lev) - st_table *tbl; - int lev; +rb_gc_save_machine_context(void) { - if (!tbl) return; - st_foreach(tbl, mark_entry, lev+1); + rb_thread_t *thread = GET_THREAD(); + + RB_VM_SAVE_MACHINE_CONTEXT(thread); +} + + +static void +mark_current_machine_context(const rb_execution_context_t *ec) +{ + rb_gc_mark_machine_context(ec); } +#endif void -rb_mark_tbl(tbl) - st_table *tbl; +rb_gc_mark_machine_context(const rb_execution_context_t *ec) { - mark_tbl(tbl, 0); + VALUE *stack_start, *stack_end; + + GET_STACK_BOUNDS(stack_start, stack_end, 0); + RUBY_DEBUG_LOG("ec->th:%u stack_start:%p stack_end:%p", rb_ec_thread_ptr(ec)->serial, stack_start, stack_end); + + void *data = +#ifdef RUBY_ASAN_ENABLED + /* gc_mark_machine_stack_location_maybe() uses data as const */ + (rb_execution_context_t *)ec; +#else + NULL; +#endif + + each_location_ptr(stack_start, stack_end, gc_mark_machine_stack_location_maybe, data); + int num_regs = sizeof(ec->machine.regs)/(sizeof(VALUE)); + each_location((VALUE*)&ec->machine.regs, num_regs, gc_mark_machine_stack_location_maybe, data); } static int -mark_keyvalue(key, value, lev) - VALUE key; - VALUE value; - int lev; +rb_mark_tbl_i(st_data_t key, st_data_t value, st_data_t data) { - gc_mark(key, lev); - gc_mark(value, lev); + gc_mark_and_pin_internal((VALUE)value); + return ST_CONTINUE; } void -mark_hash(tbl, lev) - st_table *tbl; - int lev; +rb_mark_tbl(st_table *tbl) { - if (!tbl) return; - st_foreach(tbl, mark_keyvalue, lev+1); + if (!tbl || tbl->num_entries == 0) return; + + st_foreach(tbl, rb_mark_tbl_i, 0); +} + +static void +gc_mark_tbl_no_pin(st_table *tbl) +{ + if (!tbl || tbl->num_entries == 0) return; + + st_foreach(tbl, gc_mark_tbl_no_pin_i, 0); } void -rb_mark_hash(tbl) - st_table *tbl; +rb_mark_tbl_no_pin(st_table *tbl) { - mark_hash(tbl, 0); + gc_mark_tbl_no_pin(tbl); } void -rb_gc_mark_maybe(obj) - VALUE obj; +rb_gc_mark_set_no_pin(st_table *tbl) { - if (is_pointer_to_heap((void *)obj)) { - gc_mark(obj, 0); - } + if (!tbl || tbl->num_entries == 0) return; + + st_foreach(tbl, gc_mark_set_no_pin_i, 0); +} + +static bool +gc_declarative_marking_p(const rb_data_type_t *type) +{ + return (type->flags & RUBY_TYPED_DECL_MARKING) != 0; } -#define GC_LEVEL_MAX 250 +rb_execution_context_t * +rb_gc_get_ec(void) +{ + void *objspace = rb_gc_get_objspace(); + + if (RB_LIKELY(rb_gc_impl_during_gc_p(objspace))) { + return rb_gc_impl_get_vm_context(objspace)->ec; + } + else { + return GET_EC(); + } +} void -gc_mark(ptr, lev) - VALUE ptr; - int lev; +rb_gc_mark_roots(void *objspace, const char **categoryp) { - register RVALUE *obj; + rb_execution_context_t *ec = rb_gc_get_ec(); + rb_vm_t *vm = rb_ec_vm_ptr(ec); - obj = RANY(ptr); - if (rb_special_const_p(ptr)) return; /* special const not marked */ - if (obj->as.basic.flags == 0) return; /* free cell */ - if (obj->as.basic.flags & FL_MARK) return; /* already marked */ - obj->as.basic.flags |= FL_MARK; +#define MARK_CHECKPOINT(category) do { \ + if (categoryp) *categoryp = category; \ +} while (0) + + MARK_CHECKPOINT("vm"); + rb_vm_mark(vm); + + MARK_CHECKPOINT("end_proc"); + rb_mark_end_proc(); - if (lev > GC_LEVEL_MAX) { - if (!mark_stack_overflow) { - if (mark_stack_ptr - mark_stack < MARK_STACK_MAX) { - *mark_stack_ptr = ptr; - mark_stack_ptr++; - } - else { - mark_stack_overflow = 1; - } - } - return; + MARK_CHECKPOINT("global_tbl"); + rb_gc_mark_global_tbl(); + +#if USE_YJIT + void rb_yjit_root_mark(void); // in Rust + + if (rb_yjit_enabled_p) { + MARK_CHECKPOINT("YJIT"); + rb_yjit_root_mark(); + } +#endif + +#if USE_ZJIT + void rb_zjit_root_mark(void); + if (rb_zjit_enabled_p) { + MARK_CHECKPOINT("ZJIT"); + rb_zjit_root_mark(); } - gc_mark_children(ptr, lev); +#endif + + MARK_CHECKPOINT("machine_context"); + mark_current_machine_context(ec); + + MARK_CHECKPOINT("global_symbols"); + rb_sym_global_symbols_mark_and_move(); + + MARK_CHECKPOINT("finish"); + +#undef MARK_CHECKPOINT } -void -rb_gc_mark(ptr) - VALUE ptr; +struct gc_mark_classext_foreach_arg { + rb_objspace_t *objspace; + VALUE obj; +}; + +static void +gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) { - gc_mark(ptr, 0); + struct gc_mark_classext_foreach_arg *foreach_arg = (struct gc_mark_classext_foreach_arg *)arg; + rb_objspace_t *objspace = foreach_arg->objspace; + + if (RCLASSEXT_SUPER(ext)) { + gc_mark_internal(RCLASSEXT_SUPER(ext)); + } + mark_m_tbl(objspace, RCLASSEXT_M_TBL(ext)); + + if (!rb_gc_checking_shareable()) { + // unshareable + gc_mark_internal(RCLASSEXT_FIELDS_OBJ(ext)); + gc_mark_internal(RCLASSEXT_CVC_TBL(ext)); + } + + if (!RCLASSEXT_SHARED_CONST_TBL(ext) && RCLASSEXT_CONST_TBL(ext)) { + mark_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext)); + } + mark_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext)); + gc_mark_internal(RCLASSEXT_CC_TBL(ext)); + if (RCLASSEXT_SUBCLASSES(ext)) { + gc_mark_internal(RCLASSEXT_SUBCLASSES(ext)); + } + gc_mark_internal(RCLASSEXT_CLASSPATH(ext)); } static void -gc_mark_children(ptr, lev) - VALUE ptr; - int lev; +gc_mark_classext_iclass(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) { - register RVALUE *obj = RANY(ptr); + struct gc_mark_classext_foreach_arg *foreach_arg = (struct gc_mark_classext_foreach_arg *)arg; + rb_objspace_t *objspace = foreach_arg->objspace; - goto marking; /* skip */ + if (RCLASSEXT_SUPER(ext)) { + gc_mark_internal(RCLASSEXT_SUPER(ext)); + } + if (RCLASSEXT_ICLASS_IS_ORIGIN(ext) && !RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext)) { + mark_m_tbl(objspace, RCLASSEXT_M_TBL(ext)); + } + if (RCLASSEXT_INCLUDER(ext)) { + gc_mark_internal(RCLASSEXT_INCLUDER(ext)); + } + mark_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext)); + gc_mark_internal(RCLASSEXT_CC_TBL(ext)); + if (RCLASSEXT_SUBCLASSES(ext)) { + gc_mark_internal(RCLASSEXT_SUBCLASSES(ext)); + } +} - again: - obj = RANY(ptr); - if (rb_special_const_p(ptr)) return; /* special const not marked */ - if (obj->as.basic.flags == 0) return; /* free cell */ - if (obj->as.basic.flags & FL_MARK) return; /* already marked */ - obj->as.basic.flags |= FL_MARK; +#define TYPED_DATA_REFS_OFFSET_LIST(d) (size_t *)(uintptr_t)RTYPEDDATA_TYPE(d)->function.dmark - marking: - if (FL_TEST(obj, FL_EXIVAR)) { - rb_mark_generic_ivar(ptr); +void +rb_gc_move_obj_during_marking(VALUE from, VALUE to) +{ + if (rb_obj_using_gen_fields_table_p(to)) { + rb_mark_generic_ivar(from); } +} + +void +rb_gc_mark_children(void *objspace, VALUE obj) +{ + struct gc_mark_classext_foreach_arg foreach_args; + + if (rb_obj_using_gen_fields_table_p(obj)) { + rb_mark_generic_ivar(obj); + } + + switch (BUILTIN_TYPE(obj)) { + case T_FLOAT: + case T_BIGNUM: + return; - switch (obj->as.basic.flags & T_MASK) { case T_NIL: case T_FIXNUM: - rb_bug("rb_gc_mark() called for broken object"); - break; + rb_bug("rb_gc_mark() called for broken object"); + break; case T_NODE: - mark_source_filename(obj->as.node.nd_file); - switch (nd_type(obj)) { - case NODE_IF: /* 1,2,3 */ - case NODE_FOR: - case NODE_ITER: - case NODE_CREF: - case NODE_WHEN: - case NODE_MASGN: - case NODE_RESCUE: - case NODE_RESBODY: - case NODE_CLASS: - gc_mark((VALUE)obj->as.node.u2.node, lev); - /* fall through */ - case NODE_BLOCK: /* 1,3 */ - case NODE_ARRAY: - case NODE_DSTR: - case NODE_DXSTR: - case NODE_DREGX: - case NODE_DREGX_ONCE: - case NODE_FBODY: - case NODE_ENSURE: - case NODE_CALL: - case NODE_DEFS: - case NODE_OP_ASGN1: - gc_mark((VALUE)obj->as.node.u1.node, lev); - /* fall through */ - case NODE_SUPER: /* 3 */ - case NODE_FCALL: - case NODE_DEFN: - case NODE_NEWLINE: - ptr = (VALUE)obj->as.node.u3.node; - goto again; - - case NODE_WHILE: /* 1,2 */ - case NODE_UNTIL: - case NODE_AND: - case NODE_OR: - case NODE_CASE: - case NODE_SCLASS: - case NODE_DOT2: - case NODE_DOT3: - case NODE_FLIP2: - case NODE_FLIP3: - case NODE_MATCH2: - case NODE_MATCH3: - case NODE_OP_ASGN_OR: - case NODE_OP_ASGN_AND: - case NODE_MODULE: - gc_mark((VALUE)obj->as.node.u1.node, lev); - /* fall through */ - case NODE_METHOD: /* 2 */ - case NODE_NOT: - case NODE_GASGN: - case NODE_LASGN: - case NODE_DASGN: - case NODE_DASGN_CURR: - case NODE_IASGN: - case NODE_CVDECL: - case NODE_CVASGN: - case NODE_COLON3: - case NODE_OPT_N: - case NODE_EVSTR: - ptr = (VALUE)obj->as.node.u2.node; - goto again; - - case NODE_HASH: /* 1 */ - case NODE_LIT: - case NODE_STR: - case NODE_XSTR: - case NODE_DEFINED: - case NODE_MATCH: - case NODE_RETURN: - case NODE_BREAK: - case NODE_NEXT: - case NODE_YIELD: - case NODE_COLON2: - case NODE_ARGS: - case NODE_SPLAT: - case NODE_TO_ARY: - case NODE_SVALUE: - ptr = (VALUE)obj->as.node.u1.node; - goto again; - - case NODE_SCOPE: /* 2,3 */ - case NODE_BLOCK_PASS: - case NODE_CDECL: - gc_mark((VALUE)obj->as.node.u3.node, lev); - ptr = (VALUE)obj->as.node.u2.node; - goto again; - - case NODE_ZARRAY: /* - */ - case NODE_ZSUPER: - case NODE_CFUNC: - case NODE_VCALL: - case NODE_GVAR: - case NODE_LVAR: - case NODE_DVAR: - case NODE_IVAR: - case NODE_CVAR: - case NODE_NTH_REF: - case NODE_BACK_REF: - case NODE_ALIAS: - case NODE_VALIAS: - case NODE_REDO: - case NODE_RETRY: - case NODE_UNDEF: - case NODE_SELF: - case NODE_NIL: - case NODE_TRUE: - case NODE_FALSE: - case NODE_ATTRSET: - case NODE_BLOCK_ARG: - case NODE_POSTEXE: - break; -#ifdef C_ALLOCA - case NODE_ALLOCA: - mark_locations_array((VALUE*)obj->as.node.u1.value, - obj->as.node.u3.cnt); - ptr = (VALUE)obj->as.node.u2.node; - goto again; -#endif - - default: /* unlisted NODE */ - if (is_pointer_to_heap(obj->as.node.u1.node)) { - gc_mark((VALUE)obj->as.node.u1.node, lev); - } - if (is_pointer_to_heap(obj->as.node.u2.node)) { - gc_mark((VALUE)obj->as.node.u2.node, lev); - } - if (is_pointer_to_heap(obj->as.node.u3.node)) { - gc_mark((VALUE)obj->as.node.u3.node, lev); - } - } - return; /* no need to mark class. */ - } - - gc_mark(obj->as.basic.klass, lev); - switch (obj->as.basic.flags & T_MASK) { - case T_ICLASS: + UNEXPECTED_NODE(rb_gc_mark); + break; + + case T_IMEMO: + rb_imemo_mark_and_move(obj, false); + return; + + default: + break; + } + + gc_mark_internal(RBASIC(obj)->klass); + + switch (BUILTIN_TYPE(obj)) { case T_CLASS: + if (FL_TEST_RAW(obj, FL_SINGLETON) && + !rb_gc_checking_shareable()) { + gc_mark_internal(RCLASS_ATTACHED_OBJECT(obj)); + } + // Continue to the shared T_CLASS/T_MODULE case T_MODULE: - mark_tbl(obj->as.klass.m_tbl, lev); - mark_tbl(obj->as.klass.iv_tbl, lev); - ptr = obj->as.klass.super; - goto again; + foreach_args.objspace = objspace; + foreach_args.obj = obj; + rb_class_classext_foreach(obj, gc_mark_classext_module, (void *)&foreach_args); + if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) { + gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object); + } + break; + + case T_ICLASS: + foreach_args.objspace = objspace; + foreach_args.obj = obj; + rb_class_classext_foreach(obj, gc_mark_classext_iclass, (void *)&foreach_args); + if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) { + gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object); + } + break; case T_ARRAY: - if (FL_TEST(obj, ELTS_SHARED)) { - ptr = obj->as.array.aux.shared; - goto again; - } - else { - long i, len = obj->as.array.len; - VALUE *ptr = obj->as.array.ptr; - - for (i=0; i < len; i++) { - gc_mark(*ptr++, lev); - } - } - break; + if (ARY_SHARED_P(obj)) { + VALUE root = ARY_SHARED_ROOT(obj); + gc_mark_internal(root); + } + else { + long len = RARRAY_LEN(obj); + const VALUE *ptr = RARRAY_CONST_PTR(obj); + for (long i = 0; i < len; i++) { + gc_mark_internal(ptr[i]); + } + } + break; case T_HASH: - mark_hash(obj->as.hash.tbl, lev); - ptr = obj->as.hash.ifnone; - goto again; + mark_hash(obj); + break; + + case T_SYMBOL: + gc_mark_internal(RSYMBOL(obj)->fstr); + break; case T_STRING: -#define STR_ASSOC FL_USER3 /* copied from string.c */ - if (FL_TEST(obj, ELTS_SHARED|STR_ASSOC)) { - ptr = obj->as.string.aux.shared; - goto again; - } - break; + if (STR_SHARED_P(obj)) { + if (STR_EMBED_P(RSTRING(obj)->as.heap.aux.shared)) { + /* Embedded shared strings cannot be moved because this string + * points into the slot of the shared string. There may be code + * using the RSTRING_PTR on the stack, which would pin this + * string but not pin the shared string, causing it to move. */ + gc_mark_and_pin_internal(RSTRING(obj)->as.heap.aux.shared); + } + else { + gc_mark_internal(RSTRING(obj)->as.heap.aux.shared); + } + } + break; - case T_DATA: - if (obj->as.data.dmark) (*obj->as.data.dmark)(DATA_PTR(obj)); - break; + case T_DATA: { + void *const ptr = RTYPEDDATA_GET_DATA(obj); - case T_OBJECT: - mark_tbl(obj->as.object.iv_tbl, lev); - break; + gc_mark_internal(RTYPEDDATA(obj)->fields_obj); + + if (ptr) { + if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) { + size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj); + + for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) { + gc_mark_internal(*(VALUE *)((char *)ptr + offset)); + } + } + else { + RUBY_DATA_FUNC mark_func = RTYPEDDATA_TYPE(obj)->function.dmark; + if (mark_func) (*mark_func)(ptr); + } + } + + break; + } + + case T_OBJECT: { + uint32_t len; + if (rb_obj_shape_complex_p(obj)) { + gc_mark_tbl_no_pin(ROBJECT_FIELDS_HASH(obj)); + len = ROBJECT_FIELDS_COUNT_COMPLEX(obj); + } + else { + const VALUE * const ptr = ROBJECT_FIELDS(obj); + + len = ROBJECT_FIELDS_COUNT_NOT_COMPLEX(obj); + for (uint32_t i = 0; i < len; i++) { + gc_mark_internal(ptr[i]); + } + } + break; + } case T_FILE: + if (RFILE(obj)->fptr) { + gc_mark_internal(RFILE(obj)->fptr->self); + gc_mark_internal(RFILE(obj)->fptr->pathv); + gc_mark_internal(RFILE(obj)->fptr->tied_io_for_writing); + gc_mark_internal(RFILE(obj)->fptr->writeconv_asciicompat); + gc_mark_internal(RFILE(obj)->fptr->writeconv_pre_ecopts); + gc_mark_internal(RFILE(obj)->fptr->encs.ecopts); + gc_mark_internal(RFILE(obj)->fptr->write_lock); + gc_mark_internal(RFILE(obj)->fptr->timeout); + gc_mark_internal(RFILE(obj)->fptr->wakeup_mutex); + } + break; + case T_REGEXP: - case T_FLOAT: - case T_BIGNUM: - case T_BLKTAG: - break; + gc_mark_internal(RREGEXP(obj)->src); + break; case T_MATCH: - if (obj->as.match.str) { - ptr = obj->as.match.str; - goto again; - } - break; - - case T_VARMAP: - gc_mark(obj->as.varmap.val, lev); - ptr = (VALUE)obj->as.varmap.next; - goto again; - - case T_SCOPE: - if (obj->as.scope.local_vars && (obj->as.scope.flags & SCOPE_MALLOC)) { - int n = obj->as.scope.local_tbl[0]+1; - VALUE *vars = &obj->as.scope.local_vars[-1]; - - while (n--) { - gc_mark(*vars++, lev); - } - } - break; + gc_mark_internal(RMATCH(obj)->regexp); + if (RMATCH(obj)->str) { + gc_mark_internal(RMATCH(obj)->str); + } + break; - case T_STRUCT: - { - long len = obj->as.rstruct.len; - VALUE *ptr = obj->as.rstruct.ptr; + case T_RATIONAL: + gc_mark_internal(RRATIONAL(obj)->num); + gc_mark_internal(RRATIONAL(obj)->den); + break; + + case T_COMPLEX: + gc_mark_internal(RCOMPLEX(obj)->real); + gc_mark_internal(RCOMPLEX(obj)->imag); + break; + + case T_STRUCT: { + const long len = RSTRUCT_LEN(obj); + const VALUE * const ptr = RSTRUCT_CONST_PTR(obj); + + for (long i = 0; i < len; i++) { + gc_mark_internal(ptr[i]); + } - while (len--) { - gc_mark(*ptr++, lev); - } - } - break; + if (rb_obj_shape_has_fields(obj) && !FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS)) { + gc_mark_internal(RSTRUCT_FIELDS_OBJ(obj)); + } + + break; + } default: - rb_bug("rb_gc_mark(): unknown data type 0x%lx(0x%lx) %s", - obj->as.basic.flags & T_MASK, obj, - is_pointer_to_heap(obj) ? "corrupted object" : "non object"); + if (BUILTIN_TYPE(obj) == T_MOVED) rb_bug("rb_gc_mark(): %p is T_MOVED", (void *)obj); + if (BUILTIN_TYPE(obj) == T_NONE) rb_bug("rb_gc_mark(): %p is T_NONE", (void *)obj); + if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("rb_gc_mark(): %p is T_ZOMBIE", (void *)obj); + rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s", + BUILTIN_TYPE(obj), (void *)obj, + rb_gc_impl_pointer_to_heap_p(objspace, (void *)obj) ? "corrupted object" : "non object"); + } +} + +size_t +rb_gc_obj_optimal_size(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_ARRAY: + { + size_t size = rb_ary_size_as_embedded(obj); + if (rb_gc_size_allocatable_p(size)) { + return size; + } + else { + return sizeof(struct RArray); + } + } + + case T_OBJECT: + if (rb_obj_shape_complex_p(obj)) { + return sizeof(struct RObject); + } + else { + size_t size = rb_obj_embedded_size(ROBJECT_FIELDS_CAPACITY(obj)); + if (rb_gc_size_allocatable_p(size)) { + return size; + } + else { + return sizeof(struct RObject); + } + } + + case T_STRING: + { + size_t size = rb_str_size_as_embedded(obj); + if (rb_gc_size_allocatable_p(size)) { + return size; + } + else { + return sizeof(struct RString); + } + } + + case T_HASH: + { + if (RB_OBJ_FROZEN(obj) && RHASH_AR_TABLE_P(obj)) { + return sizeof(struct RHash) + offsetof(ar_table, pairs) + RHASH_AR_TABLE_BOUND(obj) * sizeof(ar_table_pair); + } + return sizeof(struct RHash) + (RHASH_ST_TABLE_P(obj) ? sizeof(st_table) : sizeof(ar_table)); + } + + default: + return 0; + } +} + +void +rb_gc_writebarrier(VALUE a, VALUE b) +{ + rb_gc_impl_writebarrier(rb_gc_get_objspace(), a, b); +} + +void +rb_gc_writebarrier_unprotect(VALUE obj) +{ + rb_gc_impl_writebarrier_unprotect(rb_gc_get_objspace(), obj); +} + +/* + * remember `obj' if needed. + */ +void +rb_gc_writebarrier_remember(VALUE obj) +{ + rb_gc_impl_writebarrier_remember(rb_gc_get_objspace(), obj); +} + +void +rb_gc_copy_attributes(VALUE dest, VALUE obj) +{ + rb_gc_impl_copy_attributes(rb_gc_get_objspace(), dest, obj); +} + +#if USE_MODULAR_GC +int +rb_gc_modular_gc_loaded_p(void) +{ + return rb_gc_functions.modular_gc_loaded_p; +} + +const char * +rb_gc_active_gc_name(void) +{ + const char *gc_name = rb_gc_impl_active_gc_name(); + + const size_t len = strlen(gc_name); + if (len > RB_GC_MAX_NAME_LEN) { + rb_bug("GC should have a name no more than %d chars long. Currently: %zu (%s)", + RB_GC_MAX_NAME_LEN, len, gc_name); } + + return gc_name; +} +#endif + +struct rb_gc_object_metadata_entry * +rb_gc_object_metadata(VALUE obj) +{ + return rb_gc_impl_object_metadata(rb_gc_get_objspace(), obj); +} + +/* GC */ + +void * +rb_gc_ractor_cache_alloc(rb_ractor_t *ractor) +{ + return rb_gc_impl_ractor_cache_alloc(rb_gc_get_objspace(), ractor); +} + +void +rb_gc_ractor_cache_free(void *cache) +{ + rb_gc_impl_ractor_cache_free(rb_gc_get_objspace(), cache); } -static void obj_free _((VALUE)); +void +rb_gc_register_mark_object(VALUE obj) +{ + if (!rb_gc_impl_pointer_to_heap_p(rb_gc_get_objspace(), (void *)obj)) + return; + + rb_vm_register_global_object(obj); +} + +void +rb_gc_register_address(VALUE *addr) +{ + rb_vm_t *vm = GET_VM(); + + VALUE obj = *addr; + + RB_VM_LOCKING() { + if (vm->global_object_list_size == vm->global_object_list_capa) { + size_t new_capa = vm->global_object_list_capa ? vm->global_object_list_capa * 2 : 64; + SIZED_REALLOC_N(vm->global_object_list, VALUE *, new_capa, vm->global_object_list_capa); + vm->global_object_list_capa = new_capa; + } + + vm->global_object_list[vm->global_object_list_size++] = addr; + } + + /* + * Because some C extensions have assignment-then-register bugs, + * we guard `obj` here so that it would not get swept defensively. + */ + RB_GC_GUARD(obj); + if (0 && !SPECIAL_CONST_P(obj)) { + rb_warn("Object is assigned to registering address already: %"PRIsVALUE, + rb_obj_class(obj)); + rb_print_backtrace(stderr); + } +} + +void +rb_gc_unregister_address(VALUE *addr) +{ + rb_vm_t *vm = GET_VM(); + RB_VM_LOCKING() { + size_t index; + for (index = 0; index < vm->global_object_list_size; index++) { + if (addr == vm->global_object_list[index]) { + MEMMOVE( + vm->global_object_list[index], + &vm->global_object_list[index + 1], + VALUE *, + vm->global_object_list_size - index - 1 + ); + vm->global_object_list_size--; + break; + } + } + } +} + +void +rb_global_variable(VALUE *var) +{ + rb_gc_register_address(var); +} + +static VALUE +gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact) +{ + rb_gc_impl_start(rb_gc_get_objspace(), RTEST(full_mark), RTEST(immediate_mark), RTEST(immediate_sweep), RTEST(compact)); + + return Qnil; +} + +/* + * rb_objspace_each_objects() is special C API to walk through + * Ruby object space. This C API is too difficult to use it. + * To be frank, you should not use it. Or you need to read the + * source code of this function and understand what this function does. + * + * 'callback' will be called several times (the number of heap page, + * at current implementation) with: + * vstart: a pointer to the first living object of the heap_page. + * vend: a pointer to next to the valid heap_page area. + * stride: a distance to next VALUE. + * + * If callback() returns non-zero, the iteration will be stopped. + * + * This is a sample callback code to iterate liveness objects: + * + * static int + * sample_callback(void *vstart, void *vend, int stride, void *data) + * { + * VALUE v = (VALUE)vstart; + * for (; v != (VALUE)vend; v += stride) { + * if (!rb_objspace_internal_object_p(v)) { // liveness check + * // do something with live object 'v' + * } + * } + * return 0; // continue to iteration + * } + * + * Note: 'vstart' is not a top of heap_page. This point the first + * living object to grasp at least one object to avoid GC issue. + * This means that you can not walk through all Ruby object page + * including freed object page. + * + * Note: On this implementation, 'stride' is the same as sizeof(RVALUE). + * However, there are possibilities to pass variable values with + * 'stride' with some reasons. You must use stride instead of + * use some constant value in the iteration. + */ +void +rb_objspace_each_objects(int (*callback)(void *, void *, size_t, void *), void *data) +{ + rb_gc_impl_each_objects(rb_gc_get_objspace(), callback, data); +} static void -gc_sweep() -{ - RVALUE *p, *pend, *final_list; - int freed = 0; - int i, j; - unsigned long live = 0; - - if (ruby_in_compile && ruby_parser_stack_on_heap()) { - /* should not reclaim nodes during compilation - if yacc's semantic stack is not allocated on machine stack */ - for (i = 0; i < heaps_used; i++) { - p = heaps[i].slot; pend = p + heaps[i].limit; - while (p < pend) { - if (!(p->as.basic.flags&FL_MARK) && BUILTIN_TYPE(p) == T_NODE) - gc_mark((VALUE)p, 0); - p++; - } - } - } - - mark_source_filename(ruby_sourcefile); - st_foreach(source_filenames, sweep_source_filename, 0); - - freelist = 0; - final_list = deferred_final_list; - deferred_final_list = 0; - for (i = 0; i < heaps_used; i++) { - int n = 0; - RVALUE *free = freelist; - RVALUE *final = final_list; - - p = heaps[i].slot; pend = p + heaps[i].limit; - while (p < pend) { - if (!(p->as.basic.flags & FL_MARK)) { - if (p->as.basic.flags) { - obj_free((VALUE)p); - } - if (need_call_final && FL_TEST(p, FL_FINALIZE)) { - p->as.free.flags = FL_MARK; /* remain marked */ - p->as.free.next = final_list; - final_list = p; - } - else { - p->as.free.flags = 0; - p->as.free.next = freelist; - freelist = p; - } - n++; - } - else if (RBASIC(p)->flags == FL_MARK) { - /* objects to be finalized */ - /* do notning remain marked */ - } - else { - RBASIC(p)->flags &= ~FL_MARK; - live++; - } - p++; - } - if (n == heaps[i].limit && freed > FREE_MIN) { - RVALUE *pp; - - heaps[i].limit = 0; - for (pp = final_list; pp != final; pp = pp->as.free.next) { - p->as.free.flags |= FL_SINGLETON; /* freeing page mark */ - } - freelist = free; /* cancel this page from freelist */ - } - else { - freed += n; - } - } - if (malloc_increase > malloc_limit) { - malloc_limit += (malloc_increase - malloc_limit) * (double)live / (live + freed); - if (malloc_limit < GC_MALLOC_LIMIT) malloc_limit = GC_MALLOC_LIMIT; - } - malloc_increase = 0; - if (freed < FREE_MIN) { - add_heap(); - } - during_gc = 0; - - /* clear finalization list */ - if (final_list) { - RVALUE *tmp; - - if (rb_prohibit_interrupt || ruby_in_compile) { - deferred_final_list = final_list; - return; - } - - for (p = final_list; p; p = tmp) { - tmp = p->as.free.next; - run_final((VALUE)p); - if (!FL_TEST(p, FL_SINGLETON)) { /* not freeing page */ - p->as.free.flags = 0; - p->as.free.next = freelist; - freelist = p; - } - } - } - for (i = j = 1; j < heaps_used; i++) { - if (heaps[i].limit == 0) { - free(heaps[i].slot); - heaps_used--; - } - else { - if (i != j) { - heaps[j] = heaps[i]; - } - j++; - } - } -} - -void -rb_gc_force_recycle(p) - VALUE p; -{ - RANY(p)->as.free.flags = 0; - RANY(p)->as.free.next = freelist; - freelist = RANY(p); +gc_ref_update_array(void *objspace, VALUE v) +{ + if (ARY_SHARED_P(v)) { + VALUE old_root = RARRAY(v)->as.heap.aux.shared_root; + + UPDATE_IF_MOVED(objspace, RARRAY(v)->as.heap.aux.shared_root); + + VALUE new_root = RARRAY(v)->as.heap.aux.shared_root; + // If the root is embedded and its location has changed + if (ARY_EMBED_P(new_root) && new_root != old_root) { + size_t offset = (size_t)(RARRAY(v)->as.heap.ptr - RARRAY(old_root)->as.ary); + GC_ASSERT(RARRAY(v)->as.heap.ptr >= RARRAY(old_root)->as.ary); + RARRAY(v)->as.heap.ptr = RARRAY(new_root)->as.ary + offset; + } + } + else { + long len = RARRAY_LEN(v); + + if (len > 0) { + VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v); + for (long i = 0; i < len; i++) { + UPDATE_IF_MOVED(objspace, ptr[i]); + } + } + + if (rb_gc_obj_slot_size(v) >= rb_ary_size_as_embedded(v)) { + if (rb_ary_embeddable_p(v)) { + rb_ary_make_embedded(v); + } + } + } } static void -obj_free(obj) - VALUE obj; +gc_ref_update_object(void *objspace, VALUE v) { - switch (RANY(obj)->as.basic.flags & T_MASK) { - case T_NIL: - case T_FIXNUM: - case T_TRUE: - case T_FALSE: - rb_bug("obj_free() called for broken object"); - break; + VALUE *ptr = ROBJECT_FIELDS(v); + + if (FL_TEST_RAW(v, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(v)) { + gc_ref_update_table_values_only(ROBJECT_FIELDS_HASH(v)); + return; + } + + size_t slot_size = rb_gc_obj_slot_size(v); + size_t embed_size = rb_obj_embedded_size(ROBJECT_FIELDS_CAPACITY(v)); + if (slot_size >= embed_size) { + // Object can be re-embedded + memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * ROBJECT_FIELDS_COUNT(v)); + SIZED_FREE_N(ptr, ROBJECT_FIELDS_CAPACITY(v)); + FL_UNSET_RAW(v, ROBJECT_HEAP); + ptr = ROBJECT(v)->as.ary; + } } - if (FL_TEST(obj, FL_EXIVAR)) { - rb_free_generic_ivar((VALUE)obj); + for (uint32_t i = 0; i < ROBJECT_FIELDS_COUNT(v); i++) { + UPDATE_IF_MOVED(objspace, ptr[i]); } +} - switch (RANY(obj)->as.basic.flags & T_MASK) { - case T_OBJECT: - if (RANY(obj)->as.object.iv_tbl) { - st_free_table(RANY(obj)->as.object.iv_tbl); - } - break; - case T_MODULE: - case T_CLASS: - rb_clear_cache_by_class((VALUE)obj); - st_free_table(RANY(obj)->as.klass.m_tbl); - if (RANY(obj)->as.object.iv_tbl) { - st_free_table(RANY(obj)->as.object.iv_tbl); - } - break; - case T_STRING: - if (RANY(obj)->as.string.ptr && !FL_TEST(obj, ELTS_SHARED)) { - RUBY_CRITICAL(free(RANY(obj)->as.string.ptr)); - } - break; - case T_ARRAY: - if (RANY(obj)->as.array.ptr && !FL_TEST(obj, ELTS_SHARED)) { - RUBY_CRITICAL(free(RANY(obj)->as.array.ptr)); - } - break; - case T_HASH: - if (RANY(obj)->as.hash.tbl) { - st_free_table(RANY(obj)->as.hash.tbl); - } - break; - case T_REGEXP: - if (RANY(obj)->as.regexp.ptr) { - re_free_pattern(RANY(obj)->as.regexp.ptr); - } - if (RANY(obj)->as.regexp.str) { - RUBY_CRITICAL(free(RANY(obj)->as.regexp.str)); - } - break; - case T_DATA: - if (DATA_PTR(obj)) { - if ((long)RANY(obj)->as.data.dfree == -1) { - RUBY_CRITICAL(free(DATA_PTR(obj))); - } - else if (RANY(obj)->as.data.dfree) { - (*RANY(obj)->as.data.dfree)(DATA_PTR(obj)); - } - } - break; - case T_MATCH: - if (RANY(obj)->as.match.regs) { - re_free_registers(RANY(obj)->as.match.regs); - RUBY_CRITICAL(free(RANY(obj)->as.match.regs)); - } - break; - case T_FILE: - if (RANY(obj)->as.file.fptr) { - rb_io_fptr_finalize(RANY(obj)->as.file.fptr); - RUBY_CRITICAL(free(RANY(obj)->as.file.fptr)); - } - break; - case T_ICLASS: - /* iClass shares table with the module */ - break; +void +rb_gc_ref_update_table_values_only(st_table *tbl) +{ + gc_ref_update_table_values_only(tbl); +} - case T_FLOAT: - case T_VARMAP: - case T_BLKTAG: - break; +/* Update MOVED references in a VALUE=>VALUE st_table */ +void +rb_gc_update_tbl_refs(st_table *ptr) +{ + gc_update_table_refs(ptr); +} - case T_BIGNUM: - if (RANY(obj)->as.bignum.digits) { - RUBY_CRITICAL(free(RANY(obj)->as.bignum.digits)); - } - break; - case T_NODE: - switch (nd_type(obj)) { - case NODE_SCOPE: - if (RANY(obj)->as.node.u1.tbl) { - RUBY_CRITICAL(free(RANY(obj)->as.node.u1.tbl)); - } - break; -#ifdef C_ALLOCA - case NODE_ALLOCA: - RUBY_CRITICAL(free(RANY(obj)->as.node.u1.node)); - break; -#endif - } - return; /* no need to free iv_tbl */ - - case T_SCOPE: - if (RANY(obj)->as.scope.local_vars && - RANY(obj)->as.scope.flags != SCOPE_ALLOCA) { - VALUE *vars = RANY(obj)->as.scope.local_vars-1; - if (vars[0] == 0) - RUBY_CRITICAL(free(RANY(obj)->as.scope.local_tbl)); - if (RANY(obj)->as.scope.flags & SCOPE_MALLOC) - RUBY_CRITICAL(free(vars)); - } - break; +static int +rb_gc_update_set_refs_i(st_data_t key, st_data_t value, st_data_t argp, int error) +{ + if (rb_gc_location((VALUE)key) != (VALUE)key) { + return ST_REPLACE; + } - case T_STRUCT: - if (RANY(obj)->as.rstruct.ptr) { - RUBY_CRITICAL(free(RANY(obj)->as.rstruct.ptr)); - } - break; + return ST_CONTINUE; +} + +static int +rb_gc_update_set_refs_replace_i(st_data_t *key, st_data_t *value, st_data_t argp, int existing) +{ + if (rb_gc_location((VALUE)*key) != (VALUE)*key) { + *key = rb_gc_location((VALUE)*key); + } + + return ST_CONTINUE; +} + +void +rb_gc_update_set_refs(st_table *tbl) +{ + if (!tbl || tbl->num_entries == 0) return; + + if (st_foreach_with_replace(tbl, rb_gc_update_set_refs_i, rb_gc_update_set_refs_replace_i, 0)) { + rb_raise(rb_eRuntimeError, "hash modified during iteration"); + } +} + +static void +gc_ref_update_hash(void *objspace, VALUE v) +{ + rb_hash_stlike_foreach_with_replace(v, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace); +} + +static void +gc_update_values(void *objspace, long n, VALUE *values) +{ + for (long i = 0; i < n; i++) { + UPDATE_IF_MOVED(objspace, values[i]); + } +} + +void +rb_gc_update_values(long n, VALUE *values) +{ + gc_update_values(rb_gc_get_objspace(), n, values); +} + +static enum rb_id_table_iterator_result +check_id_table_move(VALUE value, void *data) +{ + void *objspace = (void *)data; + + if (gc_object_moved_p_internal(objspace, (VALUE)value)) { + return ID_TABLE_REPLACE; + } + + return ID_TABLE_CONTINUE; +} +void +rb_gc_prepare_heap_process_object(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_STRING: + // Precompute the string coderange. This both save time for when it will be + // eventually needed, and avoid mutating heap pages after a potential fork. + rb_enc_str_coderange(obj); + break; default: - rb_bug("gc_sweep(): unknown data type 0x%lx(%ld)", obj, - RANY(obj)->as.basic.flags & T_MASK); + break; } } void -rb_gc_mark_frame(frame) - struct FRAME *frame; +rb_gc_prepare_heap(void) { - mark_locations_array(frame->argv, frame->argc); - gc_mark((VALUE)frame->node, 0); + rb_gc_impl_prepare_heap(rb_gc_get_objspace()); } -#ifdef __GNUC__ -#if defined(__human68k__) || defined(DJGPP) -#if defined(__human68k__) -typedef unsigned long rb_jmp_buf[8]; -__asm__ (".even\n\ -_rb_setjmp:\n\ - move.l 4(sp),a0\n\ - movem.l d3-d7/a3-a5,(a0)\n\ - moveq.l #0,d0\n\ - rts"); -#ifdef setjmp -#undef setjmp -#endif -#else -#if defined(DJGPP) -typedef unsigned long rb_jmp_buf[6]; -__asm__ (".align 4\n\ -_rb_setjmp:\n\ - pushl %ebp\n\ - movl %esp,%ebp\n\ - movl 8(%ebp),%ebp\n\ - movl %eax,(%ebp)\n\ - movl %ebx,4(%ebp)\n\ - movl %ecx,8(%ebp)\n\ - movl %edx,12(%ebp)\n\ - movl %esi,16(%ebp)\n\ - movl %edi,20(%ebp)\n\ - popl %ebp\n\ - xorl %eax,%eax\n\ - ret"); -#endif -#endif -int rb_setjmp (rb_jmp_buf); -#define jmp_buf rb_jmp_buf -#define setjmp rb_setjmp -#endif /* __human68k__ or DJGPP */ -#endif /* __GNUC__ */ - -void -rb_gc() -{ - struct gc_list *list; - struct FRAME * volatile frame; /* gcc 2.7.2.3 -O2 bug?? */ - jmp_buf save_regs_gc_mark; - SET_STACK_END; +size_t +rb_gc_heap_id_for_size(size_t size) +{ + return rb_gc_impl_heap_id_for_size(rb_gc_get_objspace(), size); +} -#ifdef HAVE_NATIVETHREAD - if (!is_ruby_native_thread()) { - rb_bug("cross-thread violation on rb_gc()"); - } -#endif - if (dont_gc || during_gc) { - if (!freelist) { - add_heap(); - } - return; - } - if (during_gc) return; - during_gc++; - - init_mark_stack(); - - /* mark frame stack */ - for (frame = ruby_frame; frame; frame = frame->prev) { - rb_gc_mark_frame(frame); - if (frame->tmp) { - struct FRAME *tmp = frame->tmp; - while (tmp) { - rb_gc_mark_frame(tmp); - tmp = tmp->prev; - } - } - } - gc_mark((VALUE)ruby_scope, 0); - gc_mark((VALUE)ruby_dyna_vars, 0); - if (finalizer_table) { - mark_tbl(finalizer_table, 0); - } - - FLUSH_REGISTER_WINDOWS; - /* This assumes that all registers are saved into the jmp_buf (and stack) */ - setjmp(save_regs_gc_mark); - mark_locations_array((VALUE*)save_regs_gc_mark, sizeof(save_regs_gc_mark) / sizeof(VALUE *)); -#if STACK_GROW_DIRECTION < 0 - rb_gc_mark_locations((VALUE*)STACK_END, rb_gc_stack_start); -#elif STACK_GROW_DIRECTION > 0 - rb_gc_mark_locations(rb_gc_stack_start, (VALUE*)STACK_END + 1); -#else - if ((VALUE*)STACK_END < rb_gc_stack_start) - rb_gc_mark_locations((VALUE*)STACK_END, rb_gc_stack_start); - else - rb_gc_mark_locations(rb_gc_stack_start, (VALUE*)STACK_END + 1); -#endif -#ifdef __ia64__ - /* mark backing store (flushed register window on the stack) */ - /* the basic idea from guile GC code */ - { - ucontext_t ctx; - VALUE *top, *bot; - getcontext(&ctx); - mark_locations_array((VALUE*)&ctx.uc_mcontext, - ((size_t)(sizeof(VALUE)-1 + sizeof ctx.uc_mcontext)/sizeof(VALUE))); - bot = (VALUE*)__libc_ia64_register_backing_store_base; -#if defined(__FreeBSD__) - top = (VALUE*)ctx.uc_mcontext.mc_special.bspstore; -#else - top = (VALUE*)ctx.uc_mcontext.sc_ar_bsp; -#endif - rb_gc_mark_locations(bot, top); +bool +rb_gc_size_allocatable_p(size_t size) +{ + return rb_gc_impl_size_allocatable_p(size); +} + +static enum rb_id_table_iterator_result +update_id_table(VALUE *value, void *data, int existing) +{ + void *objspace = (void *)data; + + if (gc_object_moved_p_internal(objspace, (VALUE)*value)) { + *value = gc_location_internal(objspace, (VALUE)*value); } -#endif -#if defined(__human68k__) || defined(__mc68000__) - rb_gc_mark_locations((VALUE*)((char*)STACK_END + 2), - (VALUE*)((char*)rb_gc_stack_start + 2)); -#endif - rb_gc_mark_threads(); - /* mark protected global variables */ - for (list = global_List; list; list = list->next) { - rb_gc_mark_maybe(*list->varptr); + return ID_TABLE_CONTINUE; +} + +static void +update_m_tbl(void *objspace, struct rb_id_table *tbl) +{ + if (tbl) { + rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace); } - rb_mark_end_proc(); - rb_gc_mark_global_tbl(); +} - rb_mark_tbl(rb_class_tbl); - rb_gc_mark_trap_list(); +static enum rb_id_table_iterator_result +update_const_tbl_i(VALUE value, void *objspace) +{ + rb_const_entry_t *ce = (rb_const_entry_t *)value; - /* mark generic instance variables for special constants */ - rb_mark_generic_ivar_tbl(); + if (gc_object_moved_p_internal(objspace, ce->value)) { + ce->value = gc_location_internal(objspace, ce->value); + } - rb_gc_mark_parser(); - - /* gc_mark objects whose marking are not completed*/ - while (!MARK_STACK_EMPTY){ - if (mark_stack_overflow){ - gc_mark_all(); - } - else { - gc_mark_rest(); - } + if (gc_object_moved_p_internal(objspace, ce->file)) { + ce->file = gc_location_internal(objspace, ce->file); } - gc_sweep(); + + return ID_TABLE_CONTINUE; } -VALUE -rb_gc_start() +static void +update_const_tbl(void *objspace, struct rb_id_table *tbl) { - rb_gc(); - return Qnil; + if (!tbl) return; + rb_id_table_foreach_values(tbl, update_const_tbl_i, objspace); +} + +static void +update_superclasses(rb_objspace_t *objspace, rb_classext_t *ext) +{ + if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) { + size_t array_size = RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1; + for (size_t i = 0; i < array_size; i++) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPERCLASSES(ext)[i]); + } + } +} + +static void +update_classext_values(rb_objspace_t *objspace, rb_classext_t *ext, bool is_iclass) +{ + UPDATE_IF_MOVED(objspace, RCLASSEXT_ORIGIN(ext)); + UPDATE_IF_MOVED(objspace, RCLASSEXT_REFINED_CLASS(ext)); + UPDATE_IF_MOVED(objspace, RCLASSEXT_CLASSPATH(ext)); + if (is_iclass) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_INCLUDER(ext)); + } +} + +static void +update_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) +{ + struct classext_foreach_args *args = (struct classext_foreach_args *)arg; + rb_objspace_t *objspace = args->objspace; + + if (RCLASSEXT_SUPER(ext)) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPER(ext)); + } + + update_m_tbl(objspace, RCLASSEXT_M_TBL(ext)); + + UPDATE_IF_MOVED(objspace, ext->fields_obj); + if (!RCLASSEXT_SHARED_CONST_TBL(ext)) { + update_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext)); + } + UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext)); + UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext)); + update_superclasses(objspace, ext); + if (RCLASSEXT_SUBCLASSES(ext)) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext)); + } + + update_classext_values(objspace, ext, false); +} + +static void +update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) +{ + struct classext_foreach_args *args = (struct classext_foreach_args *)arg; + rb_objspace_t *objspace = args->objspace; + + if (RCLASSEXT_SUPER(ext)) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPER(ext)); + } + update_m_tbl(objspace, RCLASSEXT_M_TBL(ext)); + update_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext)); + UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext)); + UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext)); + if (RCLASSEXT_SUBCLASSES(ext)) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext)); + } + + update_classext_values(objspace, ext, true); +} + +struct global_vm_table_foreach_data { + vm_table_foreach_callback_func callback; + vm_table_update_callback_func update_callback; + void *data; + bool weak_only; +}; + +static int +vm_weak_table_foreach_weak_key(st_data_t key, st_data_t value, st_data_t data, int error) +{ + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + + int ret = iter_data->callback((VALUE)key, iter_data->data); + + if (!iter_data->weak_only) { + if (ret != ST_CONTINUE) return ret; + + ret = iter_data->callback((VALUE)value, iter_data->data); + } + + return ret; +} + +static int +vm_weak_table_foreach_update_weak_key(st_data_t *key, st_data_t *value, st_data_t data, int existing) +{ + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + + int ret = iter_data->update_callback((VALUE *)key, iter_data->data); + + if (!iter_data->weak_only) { + if (ret != ST_CONTINUE) return ret; + + ret = iter_data->update_callback((VALUE *)value, iter_data->data); + } + + return ret; +} + +static int +vm_weak_table_sym_set_foreach(VALUE *sym_ptr, void *data) +{ + VALUE sym = *sym_ptr; + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + + if (RB_SPECIAL_CONST_P(sym)) return ST_CONTINUE; + + int ret = iter_data->callback(sym, iter_data->data); + + if (ret == ST_REPLACE) { + ret = iter_data->update_callback(sym_ptr, iter_data->data); + } + + return ret; +} + +struct st_table *rb_generic_fields_tbl_get(void); + +static int +vm_weak_table_id2ref_foreach(st_data_t key, st_data_t value, st_data_t data, int error) +{ + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + + if (!iter_data->weak_only && !FIXNUM_P((VALUE)key)) { + int ret = iter_data->callback((VALUE)key, iter_data->data); + if (ret != ST_CONTINUE) return ret; + } + + return iter_data->callback((VALUE)value, iter_data->data); } +static int +vm_weak_table_id2ref_foreach_update(st_data_t *key, st_data_t *value, st_data_t data, int existing) +{ + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + + iter_data->update_callback((VALUE *)value, iter_data->data); + + if (!iter_data->weak_only && !FIXNUM_P((VALUE)*key)) { + iter_data->update_callback((VALUE *)key, iter_data->data); + } + + return ST_CONTINUE; +} + +static int +vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data) +{ + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + + int ret = iter_data->callback((VALUE)key, iter_data->data); + + VALUE new_value = (VALUE)value; + VALUE new_key = (VALUE)key; + + switch (ret) { + case ST_CONTINUE: + break; + + case ST_DELETE: + // When we're removing an object from the weak ref table, we need to + // set the shape on it so that the GC finalizer won't try to remove + // it again. A "root shape" indicates to the GC that this object + // has no fields on it, hence it won't be in the gen fields table. + RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER); + return ST_DELETE; + + case ST_REPLACE: { + ret = iter_data->update_callback(&new_key, iter_data->data); + if (key != new_key) { + ret = ST_DELETE; + } + break; + } + + default: + rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ret); + } + + if (!iter_data->weak_only) { + int ivar_ret = iter_data->callback(new_value, iter_data->data); + switch (ivar_ret) { + case ST_CONTINUE: + break; + + case ST_REPLACE: + iter_data->update_callback(&new_value, iter_data->data); + break; + + default: + rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ivar_ret); + } + } + + if (key != new_key || value != new_value) { + DURING_GC_COULD_MALLOC_REGION_START(); + { + st_insert(rb_generic_fields_tbl_get(), (st_data_t)new_key, new_value); + } + DURING_GC_COULD_MALLOC_REGION_END(); + } + + return ret; +} + +static int +vm_weak_table_frozen_strings_foreach(VALUE *str, void *data) +{ + // int retval = vm_weak_table_foreach_weak_key(key, value, data, error); + struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; + int retval = iter_data->callback(*str, iter_data->data); + + if (retval == ST_REPLACE) { + retval = iter_data->update_callback(str, iter_data->data); + } + + if (retval == ST_DELETE) { + FL_UNSET(*str, RSTRING_FSTR); + } + + return retval; +} + +void rb_fstring_foreach_with_replace(int (*callback)(VALUE *str, void *data), void *data); void -Init_stack(addr) - VALUE *addr; +rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, + vm_table_update_callback_func update_callback, + void *data, + bool weak_only, + enum rb_gc_vm_weak_tables table) { -#if defined(__human68k__) - extern void *_SEND; - rb_gc_stack_start = _SEND; -#else - if (!addr) addr = (VALUE *)&addr; - if (rb_gc_stack_start) { - if (STACK_UPPER(&addr, - rb_gc_stack_start > addr, - rb_gc_stack_start < ++addr)) - rb_gc_stack_start = addr; - return; + rb_vm_t *vm = GET_VM(); + + struct global_vm_table_foreach_data foreach_data = { + .callback = callback, + .update_callback = update_callback, + .data = data, + .weak_only = weak_only, + }; + + switch (table) { + case RB_GC_VM_CI_TABLE: { + st_foreach_with_replace( + &vm->ci_table, + vm_weak_table_foreach_weak_key, + vm_weak_table_foreach_update_weak_key, + (st_data_t)&foreach_data + ); + break; + } + case RB_GC_VM_OVERLOADED_CME_TABLE: { + st_foreach_with_replace( + &vm->overloaded_cme_table, + vm_weak_table_foreach_weak_key, + vm_weak_table_foreach_update_weak_key, + (st_data_t)&foreach_data + ); + break; + } + case RB_GC_VM_GLOBAL_SYMBOLS_TABLE: { + rb_sym_global_symbol_table_foreach_weak_reference( + vm_weak_table_sym_set_foreach, + &foreach_data + ); + break; + } + case RB_GC_VM_ID2REF_TABLE: { + if (id2ref_tbl) { + st_foreach_with_replace( + id2ref_tbl, + vm_weak_table_id2ref_foreach, + vm_weak_table_id2ref_foreach_update, + (st_data_t)&foreach_data + ); + } + break; + } + case RB_GC_VM_GENERIC_FIELDS_TABLE: { + st_table *generic_fields_tbl = rb_generic_fields_tbl_get(); + if (generic_fields_tbl) { + st_foreach( + generic_fields_tbl, + vm_weak_table_gen_fields_foreach, + (st_data_t)&foreach_data + ); + } + break; + } + case RB_GC_VM_FROZEN_STRINGS_TABLE: { + rb_fstring_foreach_with_replace( + vm_weak_table_frozen_strings_foreach, + &foreach_data + ); + break; + } + case RB_GC_VM_WEAK_TABLE_COUNT: + rb_bug("Unreachable"); + default: + rb_bug("rb_gc_vm_weak_table_foreach: unknown table %d", table); } - rb_gc_stack_start = addr; -#endif -#ifdef HAVE_GETRLIMIT - { - struct rlimit rlim; +} + +void +rb_gc_update_vm_references(void *objspace) +{ + rb_execution_context_t *ec = GET_EC(); + rb_vm_t *vm = rb_ec_vm_ptr(ec); - if (getrlimit(RLIMIT_STACK, &rlim) == 0) { - unsigned int space = rlim.rlim_cur/5; + rb_vm_update_references(vm); + rb_gc_update_global_tbl(); + rb_sym_global_symbols_mark_and_move(); - if (space > 1024*1024) space = 1024*1024; - STACK_LEVEL_MAX = (rlim.rlim_cur - space) / sizeof(VALUE); - } +#if USE_YJIT + void rb_yjit_root_update_references(void); // in Rust + + if (rb_yjit_enabled_p) { + rb_yjit_root_update_references(); } -#ifdef __ia64__ - /* ruby crashes on IA64 if compiled with optimizer on */ - /* when if STACK_LEVEL_MAX is greater than this magic number */ - /* I know this is a kludge. I suspect optimizer bug */ -#define IA64_MAGIC_STACK_LIMIT 49152 - if (STACK_LEVEL_MAX > IA64_MAGIC_STACK_LIMIT) - STACK_LEVEL_MAX = IA64_MAGIC_STACK_LIMIT; #endif + +#if USE_ZJIT + void rb_zjit_root_update_references(void); // in Rust + + if (rb_zjit_enabled_p) { + rb_zjit_root_update_references(); + } #endif } void -Init_heap() +rb_gc_update_object_references(void *objspace, VALUE obj) { - if (!rb_gc_stack_start) { - Init_stack(0); + struct classext_foreach_args args; + + switch (BUILTIN_TYPE(obj)) { + case T_CLASS: + if (FL_TEST_RAW(obj, FL_SINGLETON)) { + UPDATE_IF_MOVED(objspace, RCLASS_ATTACHED_OBJECT(obj)); + } + // Continue to the shared T_CLASS/T_MODULE + case T_MODULE: + args.klass = obj; + args.objspace = objspace; + rb_class_classext_foreach(obj, update_classext, (void *)&args); + break; + + case T_ICLASS: + args.objspace = objspace; + rb_class_classext_foreach(obj, update_iclass_classext, (void *)&args); + break; + + case T_IMEMO: + rb_imemo_mark_and_move(obj, true); + return; + + case T_NIL: + case T_FIXNUM: + case T_NODE: + case T_MOVED: + case T_NONE: + /* These can't move */ + return; + + case T_ARRAY: + gc_ref_update_array(objspace, obj); + break; + + case T_HASH: + gc_ref_update_hash(objspace, obj); + UPDATE_IF_MOVED(objspace, RHASH(obj)->ifnone); + break; + + case T_STRING: + { + if (STR_SHARED_P(obj)) { + UPDATE_IF_MOVED(objspace, RSTRING(obj)->as.heap.aux.shared); + } + + /* If, after move the string is not embedded, and can fit in the + * slot it's been placed in, then re-embed it. */ + if (rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) { + if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) { + rb_str_make_embedded(obj); + } + } + + break; + } + case T_DATA: + /* Call the compaction callback, if it exists */ + { + void *const ptr = RTYPEDDATA_GET_DATA(obj); + + UPDATE_IF_MOVED(objspace, RTYPEDDATA(obj)->fields_obj); + + if (ptr) { + if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) { + size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj); + + for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) { + VALUE *ref = (VALUE *)((char *)ptr + offset); + *ref = gc_location_internal(objspace, *ref); + } + } + else { + RUBY_DATA_FUNC compact_func = RTYPEDDATA_TYPE(obj)->function.dcompact; + if (compact_func) (*compact_func)(ptr); + } + } + } + break; + + case T_OBJECT: + gc_ref_update_object(objspace, obj); + break; + + case T_FILE: + if (RFILE(obj)->fptr) { + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->self); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->pathv); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->tied_io_for_writing); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_asciicompat); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_pre_ecopts); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->encs.ecopts); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->write_lock); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->timeout); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->wakeup_mutex); + } + break; + case T_REGEXP: + UPDATE_IF_MOVED(objspace, RREGEXP(obj)->src); + break; + + case T_SYMBOL: + UPDATE_IF_MOVED(objspace, RSYMBOL(obj)->fstr); + break; + + case T_FLOAT: + case T_BIGNUM: + break; + + case T_MATCH: + UPDATE_IF_MOVED(objspace, RMATCH(obj)->regexp); + + if (RMATCH(obj)->str) { + UPDATE_IF_MOVED(objspace, RMATCH(obj)->str); + } + break; + + case T_RATIONAL: + UPDATE_IF_MOVED(objspace, RRATIONAL(obj)->num); + UPDATE_IF_MOVED(objspace, RRATIONAL(obj)->den); + break; + + case T_COMPLEX: + UPDATE_IF_MOVED(objspace, RCOMPLEX(obj)->real); + UPDATE_IF_MOVED(objspace, RCOMPLEX(obj)->imag); + + break; + + case T_STRUCT: + { + long i, len = RSTRUCT_LEN(obj); + VALUE *ptr = (VALUE *)RSTRUCT_CONST_PTR(obj); + + for (i = 0; i < len; i++) { + UPDATE_IF_MOVED(objspace, ptr[i]); + } + + if (RSTRUCT_EMBED_LEN(obj)) { + if (!FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS)) { + UPDATE_IF_MOVED(objspace, ptr[len]); + } + } + else { + UPDATE_IF_MOVED(objspace, RSTRUCT(obj)->as.heap.fields_obj); + } + } + break; + default: + rb_bug("unreachable"); + break; } - add_heap(); + + UPDATE_IF_MOVED(objspace, RBASIC(obj)->klass); +} + +VALUE +rb_gc_start(void) +{ + rb_gc(); + return Qnil; +} + +void +rb_gc(void) +{ + unless_objspace(objspace) { return; } + + rb_gc_impl_start(objspace, true, true, true, false); +} + +int +rb_during_gc(void) +{ + unless_objspace(objspace) { return FALSE; } + + return rb_gc_impl_during_gc_p(objspace); +} + +size_t +rb_gc_count(void) +{ + return rb_gc_impl_gc_count(rb_gc_get_objspace()); } static VALUE -os_live_obj() +gc_count(rb_execution_context_t *ec, VALUE self) +{ + return SIZET2NUM(rb_gc_count()); +} + +VALUE +rb_gc_latest_gc_info(VALUE key) { - int i; - int n = 0; + if (!SYMBOL_P(key) && !RB_TYPE_P(key, T_HASH)) { + rb_raise(rb_eTypeError, "non-hash or symbol given"); + } - for (i = 0; i < heaps_used; i++) { - RVALUE *p, *pend; + VALUE val = rb_gc_impl_latest_gc_info(rb_gc_get_objspace(), key); - p = heaps[i].slot; pend = p + heaps[i].limit; - for (;p < pend; p++) { - if (p->as.basic.flags) { - switch (TYPE(p)) { - case T_ICLASS: - case T_VARMAP: - case T_SCOPE: - case T_NODE: - continue; - case T_CLASS: - if (FL_TEST(p, FL_SINGLETON)) continue; - default: - if (!p->as.basic.klass) continue; - rb_yield((VALUE)p); - n++; - } - } - } + if (val == Qundef) { + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); } - return INT2FIX(n); + return val; } static VALUE -os_obj_of(of) - VALUE of; +gc_stat(rb_execution_context_t *ec, VALUE self, VALUE arg) // arg is (nil || hash || symbol) +{ + if (NIL_P(arg)) { + arg = rb_hash_new(); + } + else if (!RB_TYPE_P(arg, T_HASH) && !SYMBOL_P(arg)) { + rb_raise(rb_eTypeError, "non-hash or symbol given"); + } + + VALUE ret = rb_gc_impl_stat(rb_gc_get_objspace(), arg); + + if (ret == Qundef) { + GC_ASSERT(SYMBOL_P(arg)); + + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg)); + } + + return ret; +} + +size_t +rb_gc_stat(VALUE arg) { - int i; - int n = 0; - - for (i = 0; i < heaps_used; i++) { - RVALUE *p, *pend; - - p = heaps[i].slot; pend = p + heaps[i].limit; - for (;p < pend; p++) { - if (p->as.basic.flags) { - switch (TYPE(p)) { - case T_ICLASS: - case T_VARMAP: - case T_SCOPE: - case T_NODE: - continue; - case T_CLASS: - if (FL_TEST(p, FL_SINGLETON)) continue; - default: - if (!p->as.basic.klass) continue; - if (rb_obj_is_kind_of((VALUE)p, of)) { - rb_yield((VALUE)p); - n++; - } - } - } - } - } - - return INT2FIX(n); + if (!RB_TYPE_P(arg, T_HASH) && !SYMBOL_P(arg)) { + rb_raise(rb_eTypeError, "non-hash or symbol given"); + } + + VALUE ret = rb_gc_impl_stat(rb_gc_get_objspace(), arg); + + if (ret == Qundef) { + GC_ASSERT(SYMBOL_P(arg)); + + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg)); + } + + if (SYMBOL_P(arg)) { + return NUM2SIZET(ret); + } + else { + return 0; + } } static VALUE -os_each_obj(argc, argv) - int argc; - VALUE *argv; +gc_stat_heap(rb_execution_context_t *ec, VALUE self, VALUE heap_name, VALUE arg) { - VALUE of; + if (NIL_P(arg)) { + arg = rb_hash_new(); + } - rb_secure(4); - if (rb_scan_args(argc, argv, "01", &of) == 0) { - return os_live_obj(); + if (NIL_P(heap_name)) { + if (!RB_TYPE_P(arg, T_HASH)) { + rb_raise(rb_eTypeError, "non-hash given"); + } + } + else if (FIXNUM_P(heap_name)) { + if (!SYMBOL_P(arg) && !RB_TYPE_P(arg, T_HASH)) { + rb_raise(rb_eTypeError, "non-hash or symbol given"); + } } else { - return os_obj_of(of); + rb_raise(rb_eTypeError, "heap_name must be nil or an Integer"); + } + + VALUE ret = rb_gc_impl_stat_heap(rb_gc_get_objspace(), heap_name, arg); + + if (ret == Qundef) { + GC_ASSERT(SYMBOL_P(arg)); + + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg)); } + + return ret; } -static VALUE finalizers; +static VALUE +gc_config_get(rb_execution_context_t *ec, VALUE self) +{ + VALUE cfg_hash = rb_gc_impl_config_get(rb_gc_get_objspace()); + rb_hash_aset(cfg_hash, sym("implementation"), rb_fstring_cstr(rb_gc_impl_active_gc_name())); + + return cfg_hash; +} static VALUE -add_final(os, block) - VALUE os, block; +gc_config_set(rb_execution_context_t *ec, VALUE self, VALUE hash) { - rb_warn("ObjectSpace::add_finalizer is deprecated; use define_finalizer"); - if (!rb_respond_to(block, rb_intern("call"))) { - rb_raise(rb_eArgError, "wrong type argument %s (should be callable)", - rb_obj_classname(block)); - } - rb_ary_push(finalizers, block); - return block; + void *objspace = rb_gc_get_objspace(); + + rb_gc_impl_config_set(objspace, hash); + + return Qnil; } static VALUE -rm_final(os, block) - VALUE os, block; +gc_stress_get(rb_execution_context_t *ec, VALUE self) { - rb_warn("ObjectSpace::remove_finalizer is deprecated; use undefine_finalizer"); - rb_ary_delete(finalizers, block); - return block; + return rb_gc_impl_stress_get(rb_gc_get_objspace()); } static VALUE -finals() +gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) +{ + rb_gc_impl_stress_set(rb_gc_get_objspace(), flag); + + return flag; +} + +void +rb_gc_initial_stress_set(VALUE flag) { - rb_warn("ObjectSpace::finalizers is deprecated"); - return finalizers; + initial_stress = flag; +} + +size_t * +rb_gc_heap_sizes(void) +{ + return rb_gc_impl_heap_sizes(rb_gc_get_objspace()); +} + +VALUE +rb_gc_enable(void) +{ + return rb_objspace_gc_enable(rb_gc_get_objspace()); +} + +VALUE +rb_objspace_gc_enable(void *objspace) +{ + bool disabled = !rb_gc_impl_gc_enabled_p(objspace); + rb_gc_impl_gc_enable(objspace); + return RBOOL(disabled); } static VALUE -call_final(os, obj) - VALUE os, obj; +gc_enable(rb_execution_context_t *ec, VALUE _) { - rb_warn("ObjectSpace::call_finalizer is deprecated; use define_finalizer"); - need_call_final = 1; - FL_SET(obj, FL_FINALIZE); - return obj; + return rb_gc_enable(); } static VALUE -undefine_final(os, obj) - VALUE os, obj; +gc_disable_no_rest(void *objspace) { - if (finalizer_table) { - st_delete(finalizer_table, (st_data_t*)&obj, 0); - } - return obj; + bool disabled = !rb_gc_impl_gc_enabled_p(objspace); + rb_gc_impl_gc_disable(objspace, false); + return RBOOL(disabled); +} + +VALUE +rb_gc_disable_no_rest(void) +{ + return gc_disable_no_rest(rb_gc_get_objspace()); +} + +VALUE +rb_gc_disable(void) +{ + return rb_objspace_gc_disable(rb_gc_get_objspace()); +} + +VALUE +rb_objspace_gc_disable(void *objspace) +{ + bool disabled = !rb_gc_impl_gc_enabled_p(objspace); + rb_gc_impl_gc_disable(objspace, true); + return RBOOL(disabled); } static VALUE -define_final(argc, argv, os) - int argc; - VALUE *argv; - VALUE os; +gc_disable(rb_execution_context_t *ec, VALUE _) { - VALUE obj, block, table; + return rb_gc_disable(); +} - rb_scan_args(argc, argv, "11", &obj, &block); - if (argc == 1) { - block = rb_block_proc(); +// TODO: think about moving ruby_gc_set_params into Init_heap or Init_gc +void +ruby_gc_set_params(void) +{ + rb_gc_impl_set_params(rb_gc_get_objspace()); +} + +void +rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data) +{ + RB_VM_LOCKING() { + if (rb_gc_impl_during_gc_p(rb_gc_get_objspace())) rb_bug("rb_objspace_reachable_objects_from() is not supported while during GC"); + + if (!RB_SPECIAL_CONST_P(obj)) { + rb_vm_t *vm = GET_VM(); + struct gc_mark_func_data_struct *prev_mfd = vm->gc.mark_func_data; + struct gc_mark_func_data_struct mfd = { + .mark_func = func, + .data = data, + }; + + vm->gc.mark_func_data = &mfd; + rb_gc_mark_children(rb_gc_get_objspace(), obj); + vm->gc.mark_func_data = prev_mfd; + } } - else if (!rb_respond_to(block, rb_intern("call"))) { - rb_raise(rb_eArgError, "wrong type argument %s (should be callable)", - rb_obj_classname(block)); +} + +struct root_objects_data { + const char *category; + void (*func)(const char *category, VALUE, void *); + void *data; +}; + +static void +root_objects_from(VALUE obj, void *ptr) +{ + const struct root_objects_data *data = (struct root_objects_data *)ptr; + (*data->func)(data->category, obj, data->data); +} + +void +rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data) +{ + if (rb_gc_impl_during_gc_p(rb_gc_get_objspace())) rb_bug("rb_gc_impl_objspace_reachable_objects_from_root() is not supported while during GC"); + + rb_vm_t *vm = GET_VM(); + + struct root_objects_data data = { + .func = func, + .data = passing_data, + }; + + struct gc_mark_func_data_struct *prev_mfd = vm->gc.mark_func_data; + struct gc_mark_func_data_struct mfd = { + .mark_func = root_objects_from, + .data = &data, + }; + + vm->gc.mark_func_data = &mfd; + rb_gc_save_machine_context(); + rb_gc_mark_roots(vm->gc.objspace, &data.category); + vm->gc.mark_func_data = prev_mfd; +} + +/* + ------------------------------ DEBUG ------------------------------ +*/ + +static const char * +type_name(int type, VALUE obj) +{ + switch (type) { +#define TYPE_NAME(t) case (t): return #t; + TYPE_NAME(T_NONE); + TYPE_NAME(T_OBJECT); + TYPE_NAME(T_CLASS); + TYPE_NAME(T_MODULE); + TYPE_NAME(T_FLOAT); + TYPE_NAME(T_STRING); + TYPE_NAME(T_REGEXP); + TYPE_NAME(T_ARRAY); + TYPE_NAME(T_HASH); + TYPE_NAME(T_STRUCT); + TYPE_NAME(T_BIGNUM); + TYPE_NAME(T_FILE); + TYPE_NAME(T_MATCH); + TYPE_NAME(T_COMPLEX); + TYPE_NAME(T_RATIONAL); + TYPE_NAME(T_NIL); + TYPE_NAME(T_TRUE); + TYPE_NAME(T_FALSE); + TYPE_NAME(T_SYMBOL); + TYPE_NAME(T_FIXNUM); + TYPE_NAME(T_UNDEF); + TYPE_NAME(T_IMEMO); + TYPE_NAME(T_ICLASS); + TYPE_NAME(T_MOVED); + TYPE_NAME(T_ZOMBIE); + case T_DATA: + if (obj && rb_objspace_data_type_name(obj)) { + return rb_objspace_data_type_name(obj); + } + return "T_DATA"; +#undef TYPE_NAME + } + return "unknown"; +} + +static const char * +obj_type_name(VALUE obj) +{ + return type_name(TYPE(obj), obj); +} + +const char * +rb_method_type_name(rb_method_type_t type) +{ + switch (type) { + case VM_METHOD_TYPE_ISEQ: return "iseq"; + case VM_METHOD_TYPE_ATTRSET: return "attrset"; + case VM_METHOD_TYPE_IVAR: return "ivar"; + case VM_METHOD_TYPE_BMETHOD: return "bmethod"; + case VM_METHOD_TYPE_ALIAS: return "alias"; + case VM_METHOD_TYPE_REFINED: return "refined"; + case VM_METHOD_TYPE_CFUNC: return "cfunc"; + case VM_METHOD_TYPE_ZSUPER: return "zsuper"; + case VM_METHOD_TYPE_MISSING: return "missing"; + case VM_METHOD_TYPE_OPTIMIZED: return "optimized"; + case VM_METHOD_TYPE_UNDEF: return "undef"; + case VM_METHOD_TYPE_NOTIMPLEMENTED: return "notimplemented"; } - need_call_final = 1; - FL_SET(obj, FL_FINALIZE); + rb_bug("rb_method_type_name: unreachable (type: %d)", type); +} - if (!finalizer_table) { - finalizer_table = st_init_numtable(); +static void +rb_raw_iseq_info(char *const buff, const size_t buff_size, const rb_iseq_t *iseq) +{ + if (buff_size > 0 && ISEQ_BODY(iseq) && ISEQ_BODY(iseq)->location.label && !RB_TYPE_P(ISEQ_BODY(iseq)->location.pathobj, T_MOVED)) { + VALUE path = rb_iseq_path(iseq); + int n = ISEQ_BODY(iseq)->location.first_lineno; + snprintf(buff, buff_size, " %s@%s:%d", + RSTRING_PTR(ISEQ_BODY(iseq)->location.label), + RSTRING_PTR(path), n); } - if (st_lookup(finalizer_table, obj, &table)) { - rb_ary_push(table, block); +} + +static int +str_len_no_raise(VALUE str) +{ + long len = RSTRING_LEN(str); + if (len < 0) return 0; + if (len > INT_MAX) return INT_MAX; + return (int)len; +} + +#define BUFF_ARGS buff + pos, buff_size - pos +#define APPEND_F(...) if ((pos += snprintf(BUFF_ARGS, "" __VA_ARGS__)) >= buff_size) goto end +#define APPEND_S(s) do { \ + if ((pos + (int)rb_strlen_lit(s)) >= buff_size) { \ + goto end; \ + } \ + else { \ + memcpy(buff + pos, (s), rb_strlen_lit(s) + 1); \ + } \ + } while (0) +#define C(c, s) ((c) != 0 ? (s) : " ") + +static size_t +rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj) +{ + size_t pos = 0; + + if (SPECIAL_CONST_P(obj)) { + APPEND_F("%s", obj_type_name(obj)); + + if (FIXNUM_P(obj)) { + APPEND_F(" %ld", FIX2LONG(obj)); + } + else if (SYMBOL_P(obj)) { + APPEND_F(" %s", rb_id2name(SYM2ID(obj))); + } } else { - st_add_direct(finalizer_table, obj, rb_ary_new3(1, block)); + // const int age = RVALUE_AGE_GET(obj); + + if (rb_gc_impl_pointer_to_heap_p(rb_gc_get_objspace(), (void *)obj)) { + APPEND_F("%p %s/", (void *)obj, obj_type_name(obj)); + // TODO: fixme + // APPEND_F("%p [%d%s%s%s%s%s%s] %s ", + // (void *)obj, age, + // C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"), + // C(RVALUE_MARK_BITMAP(obj), "M"), + // C(RVALUE_PIN_BITMAP(obj), "P"), + // C(RVALUE_MARKING_BITMAP(obj), "R"), + // C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"), + // C(rb_objspace_garbage_object_p(obj), "G"), + // obj_type_name(obj)); + } + else { + /* fake */ + // APPEND_F("%p [%dXXXX] %s", + // (void *)obj, age, + // obj_type_name(obj)); + } + + if (internal_object_p(obj)) { + /* ignore */ + } + else if (RBASIC(obj)->klass == 0) { + APPEND_S("(temporary internal)"); + } + else if (RTEST(RBASIC(obj)->klass)) { + VALUE class_path = rb_mod_name(RBASIC(obj)->klass); + if (!NIL_P(class_path)) { + APPEND_F("%s ", RSTRING_PTR(class_path)); + } + } } - return block; + end: + + return pos; } +const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj); + +static size_t +rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALUE obj, size_t pos) +{ + if (LIKELY(pos < buff_size) && !SPECIAL_CONST_P(obj)) { + const enum ruby_value_type type = BUILTIN_TYPE(obj); + + switch (type) { + case T_NODE: + UNEXPECTED_NODE(rb_raw_obj_info); + break; + case T_ARRAY: + if (ARY_SHARED_P(obj)) { + APPEND_S("shared -> "); + rb_raw_obj_info(BUFF_ARGS, ARY_SHARED_ROOT(obj)); + } + else { + APPEND_F("[%s%s%s] ", + C(ARY_EMBED_P(obj), "E"), + C(ARY_SHARED_P(obj), "S"), + C(ARY_SHARED_ROOT_P(obj), "R")); + + if (ARY_EMBED_P(obj)) { + APPEND_F("len: %ld (embed)", + RARRAY_LEN(obj)); + } + else { + APPEND_F("len: %ld, capa:%ld ptr:%p", + RARRAY_LEN(obj), + RARRAY(obj)->as.heap.aux.capa, + (void *)RARRAY_CONST_PTR(obj)); + } + } + break; + case T_STRING: { + APPEND_F("[%s%s] ", + C(FL_TEST(obj, RSTRING_FSTR), "F"), + C(RB_OBJ_FROZEN(obj), "R")); + + if (STR_SHARED_P(obj)) { + APPEND_F(" [shared] len: %ld", RSTRING_LEN(obj)); + } + else { + if (STR_EMBED_P(obj)) APPEND_S(" [embed]"); + + APPEND_F(" len: %ld, capa: %" PRIdSIZE, RSTRING_LEN(obj), rb_str_capacity(obj)); + } + APPEND_F(" \"%.*s\"", str_len_no_raise(obj), RSTRING_PTR(obj)); + break; + } + case T_SYMBOL: { + VALUE fstr = RSYMBOL(obj)->fstr; + ID id = RSYMBOL(obj)->id; + if (RB_TYPE_P(fstr, T_STRING)) { + APPEND_F(":%s id:%d", RSTRING_PTR(fstr), (unsigned int)id); + } + else { + APPEND_F("(%p) id:%d", (void *)fstr, (unsigned int)id); + } + break; + } + case T_MOVED: { + APPEND_F("-> %p", (void*)gc_location_internal(rb_gc_get_objspace(), obj)); + break; + } + case T_HASH: { + APPEND_F("[%c] %"PRIdSIZE, + RHASH_AR_TABLE_P(obj) ? 'A' : 'S', + RHASH_SIZE(obj)); + break; + } + case T_CLASS: + case T_MODULE: + { + VALUE class_path = rb_mod_name(obj); + if (!NIL_P(class_path)) { + APPEND_F("%s", RSTRING_PTR(class_path)); + } + else { + APPEND_S("(anon)"); + } + break; + } + case T_ICLASS: + { + VALUE class_path = rb_mod_name(RBASIC_CLASS(obj)); + if (!NIL_P(class_path)) { + APPEND_F("src:%s", RSTRING_PTR(class_path)); + } + break; + } + case T_OBJECT: + { + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(obj)) { + size_t hash_len = rb_st_table_size(ROBJECT_FIELDS_HASH(obj)); + APPEND_F("(complex) len:%zu", hash_len); + } + else { + APPEND_F("(embed) len:%d capa:%d", RSHAPE_LEN(RBASIC_SHAPE_ID(obj)), ROBJECT_FIELDS_CAPACITY(obj)); + } + } + else { + APPEND_F("len:%d capa:%d ptr:%p", RSHAPE_LEN(RBASIC_SHAPE_ID(obj)), ROBJECT_FIELDS_CAPACITY(obj), (void *)ROBJECT_FIELDS(obj)); + } + } + break; + case T_DATA: { + const struct rb_block *block; + const rb_iseq_t *iseq; + if (rb_obj_is_proc(obj) && + (block = vm_proc_block(obj)) != NULL && + (vm_block_type(block) == block_type_iseq) && + (iseq = vm_block_iseq(block)) != NULL) { + rb_raw_iseq_info(BUFF_ARGS, iseq); + } + else if (rb_ractor_p(obj)) { + rb_ractor_t *r = (void *)DATA_PTR(obj); + if (r) { + APPEND_F("r:%d", r->pub.id); + } + } + break; + } + case T_IMEMO: { + APPEND_F("<%s> ", rb_imemo_name(imemo_type(obj))); + + switch (imemo_type(obj)) { + case imemo_ment: + { + const rb_method_entry_t *me = (const rb_method_entry_t *)obj; + + APPEND_F(":%s (%s%s%s%s) type:%s aliased:%d owner:%p defined_class:%p", + rb_id2name(me->called_id), + METHOD_ENTRY_VISI(me) == METHOD_VISI_PUBLIC ? "pub" : + METHOD_ENTRY_VISI(me) == METHOD_VISI_PRIVATE ? "pri" : "pro", + METHOD_ENTRY_COMPLEMENTED(me) ? ",cmp" : "", + METHOD_ENTRY_CACHED(me) ? ",cc" : "", + METHOD_ENTRY_INVALIDATED(me) ? ",inv" : "", + me->def ? rb_method_type_name(me->def->type) : "NULL", + me->def ? me->def->aliased : -1, + (void *)me->owner, // obj_info(me->owner), + (void *)me->defined_class); //obj_info(me->defined_class))); + + if (me->def) { + switch (me->def->type) { + case VM_METHOD_TYPE_ISEQ: + APPEND_S(" (iseq:"); + rb_raw_obj_info(BUFF_ARGS, (VALUE)me->def->body.iseq.iseqptr); + APPEND_S(")"); + break; + default: + break; + } + } + + break; + } + case imemo_iseq: { + const rb_iseq_t *iseq = (const rb_iseq_t *)obj; + rb_raw_iseq_info(BUFF_ARGS, iseq); + break; + } + case imemo_callinfo: + { + const struct rb_callinfo *ci = (const struct rb_callinfo *)obj; + APPEND_F("(mid:%s, flag:%x argc:%d, kwarg:%s)", + rb_id2name(vm_ci_mid(ci)), + vm_ci_flag(ci), + vm_ci_argc(ci), + vm_ci_kwarg(ci) ? "available" : "NULL"); + break; + } + case imemo_callcache: + { + const struct rb_callcache *cc = (const struct rb_callcache *)obj; + VALUE class_path = vm_cc_valid(cc) ? rb_mod_name(cc->klass) : Qnil; + const rb_callable_method_entry_t *cme = vm_cc_cme(cc); + + APPEND_F("(klass:%s cme:%s%s (%p) call:%p", + NIL_P(class_path) ? (vm_cc_valid(cc) ? "??" : "<NULL>") : RSTRING_PTR(class_path), + cme ? rb_id2name(cme->called_id) : "<NULL>", + cme ? (METHOD_ENTRY_INVALIDATED(cme) ? " [inv]" : "") : "", + (void *)cme, + (void *)(uintptr_t)vm_cc_call(cc)); + break; + } + default: + break; + } + } + default: + break; + } + } + end: + + return pos; +} + +#undef C + +#ifdef RUBY_ASAN_ENABLED void -rb_gc_copy_finalizer(dest, obj) - VALUE dest, obj; +rb_asan_poison_object(VALUE obj) { - VALUE table; + MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj; + asan_poison_memory_region(ptr, rb_gc_obj_slot_size(obj)); +} - if (!finalizer_table) return; - if (!FL_TEST(obj, FL_FINALIZE)) return; - if (FL_TEST(dest, FL_FINALIZE)) { - rb_warn("copy_finalizer: descarding old finalizers"); +void +rb_asan_unpoison_object(VALUE obj, bool newobj_p) +{ + MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj; + asan_unpoison_memory_region(ptr, rb_gc_obj_slot_size(obj), newobj_p); +} + +void * +rb_asan_poisoned_object_p(VALUE obj) +{ + MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj; + return __asan_region_is_poisoned(ptr, rb_gc_obj_slot_size(obj)); +} +#endif + +static void +raw_obj_info(char *const buff, const size_t buff_size, VALUE obj) +{ + size_t pos = rb_raw_obj_info_common(buff, buff_size, obj); + pos = rb_raw_obj_info_buitin_type(buff, buff_size, obj, pos); + if (pos >= buff_size) {} // truncated +} + +const char * +rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj) +{ + void *objspace = rb_gc_get_objspace(); + + if (SPECIAL_CONST_P(obj)) { + raw_obj_info(buff, buff_size, obj); } - if (st_lookup(finalizer_table, obj, &table)) { - st_insert(finalizer_table, dest, table); + else if (!rb_gc_impl_pointer_to_heap_p(objspace, (const void *)obj)) { + snprintf(buff, buff_size, "out-of-heap:%p", (void *)obj); } +#if 0 // maybe no need to check it? + else if (0 && rb_gc_impl_garbage_object_p(objspace, obj)) { + snprintf(buff, buff_size, "garbage:%p", (void *)obj); + } +#endif + else { + asan_unpoisoning_object(obj) { + raw_obj_info(buff, buff_size, obj); + } + } + return buff; } -static VALUE -run_single_final(args) - VALUE *args; +#undef APPEND_S +#undef APPEND_F +#undef BUFF_ARGS + +/* Increments *var atomically and resets *var to 0 when maxval is + * reached. Returns the wraparound old *var value (0...maxval). */ +static rb_atomic_t +atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval) { - rb_eval_cmd(args[0], args[1], 0); - return Qnil; + rb_atomic_t oldval = RUBY_ATOMIC_FETCH_ADD(*var, 1); + if (RB_UNLIKELY(oldval >= maxval - 1)) { // wraparound *var + const rb_atomic_t newval = oldval + 1; + RUBY_ATOMIC_CAS(*var, newval, newval % maxval); + oldval %= maxval; + } + return oldval; +} + +static const char * +obj_info(VALUE obj) +{ + if (RGENGC_OBJ_INFO) { + static struct { + rb_atomic_t index; + char buffers[10][0x100]; + } info = {0}; + + rb_atomic_t index = atomic_inc_wraparound(&info.index, numberof(info.buffers)); + char *const buff = info.buffers[index]; + return rb_raw_obj_info(buff, sizeof(info.buffers[0]), obj); + } + return obj_type_name(obj); +} + +/* + ------------------------ Extended allocator ------------------------ +*/ + +struct gc_raise_tag { + VALUE exc; + const char *fmt; + va_list *ap; +}; + +static void * +gc_vraise(void *ptr) +{ + struct gc_raise_tag *argv = ptr; + rb_vraise(argv->exc, argv->fmt, *argv->ap); + UNREACHABLE_RETURN(NULL); } static void -run_final(obj) - VALUE obj; +gc_raise(VALUE exc, const char *fmt, ...) { - long i; - int status, critical_save = rb_thread_critical; - VALUE args[2], table; - - rb_thread_critical = Qtrue; - args[1] = rb_ary_new3(1, rb_obj_id(obj)); /* make obj into id */ - for (i=0; i<RARRAY(finalizers)->len; i++) { - args[0] = RARRAY(finalizers)->ptr[i]; - rb_protect((VALUE(*)_((VALUE)))run_single_final, (VALUE)args, &status); - } - if (finalizer_table && st_delete(finalizer_table, (st_data_t*)&obj, &table)) { - for (i=0; i<RARRAY(table)->len; i++) { - args[0] = RARRAY(table)->ptr[i]; - rb_protect((VALUE(*)_((VALUE)))run_single_final, (VALUE)args, &status); - } - } - rb_thread_critical = critical_save; -} - -void -rb_gc_call_finalizer_at_exit() -{ - RVALUE *p, *pend; - int i; - - /* run finalizers */ - if (need_call_final) { - if (deferred_final_list) { - p = deferred_final_list; - while (p) { - RVALUE *tmp = p; - p = p->as.free.next; - run_final((VALUE)tmp); - } - } - for (i = 0; i < heaps_used; i++) { - p = heaps[i].slot; pend = p + heaps[i].limit; - while (p < pend) { - if (FL_TEST(p, FL_FINALIZE)) { - FL_UNSET(p, FL_FINALIZE); - p->as.basic.klass = 0; - run_final((VALUE)p); - } - p++; - } - } - } - /* run data object's finalizers */ - for (i = 0; i < heaps_used; i++) { - p = heaps[i].slot; pend = p + heaps[i].limit; - while (p < pend) { - if (BUILTIN_TYPE(p) == T_DATA && - DATA_PTR(p) && RANY(p)->as.data.dfree) { - p->as.free.flags = 0; - if ((long)RANY(p)->as.data.dfree == -1) { - RUBY_CRITICAL(free(DATA_PTR(p))); - } - else if (RANY(p)->as.data.dfree) { - (*RANY(p)->as.data.dfree)(DATA_PTR(p)); - } - } - else if (BUILTIN_TYPE(p) == T_FILE) { - p->as.free.flags = 0; - rb_io_fptr_finalize(RANY(p)->as.file.fptr); - } - p++; - } + va_list ap; + va_start(ap, fmt); + struct gc_raise_tag argv = { + exc, fmt, &ap, + }; + + if (ruby_native_thread_p()) { + rb_thread_call_with_gvl(gc_vraise, &argv); + UNREACHABLE; + } + else { + /* Not in a ruby thread */ + fprintf(stderr, "%s", "[FATAL] "); + vfprintf(stderr, fmt, ap); } + + va_end(ap); + abort(); } -static VALUE -id2ref(obj, id) - VALUE obj, id; +NORETURN(static void negative_size_allocation_error(const char *)); +static void +negative_size_allocation_error(const char *msg) +{ + gc_raise(rb_eNoMemError, "%s", msg); +} + +static void * +ruby_memerror_body(void *dummy) +{ + rb_memerror(); + return 0; +} + +NORETURN(static void ruby_memerror(void)); +RBIMPL_ATTR_MAYBE_UNUSED() +static void +ruby_memerror(void) +{ + if (ruby_thread_has_gvl_p()) { + rb_memerror(); + } + else { + if (ruby_native_thread_p()) { + rb_thread_call_with_gvl(ruby_memerror_body, 0); + } + else { + /* no ruby thread */ + fprintf(stderr, "[FATAL] failed to allocate memory\n"); + } + } + + /* We have discussions whether we should die here; */ + /* We might rethink about it later. */ + exit(EXIT_FAILURE); +} + +void +rb_memerror(void) +{ + /* the `GET_VM()->special_exceptions` below assumes that + * the VM is reachable from the current thread. We should + * definitely make sure of that. */ + RUBY_ASSERT_ALWAYS(ruby_thread_has_gvl_p()); + + rb_execution_context_t *ec = GET_EC(); + VALUE exc = GET_VM()->special_exceptions[ruby_error_nomemory]; + + if (!exc || + rb_ec_raised_p(ec, RAISED_NOMEMORY) || + rb_ec_vm_lock_rec(ec) != ec->tag->lock_rec) { + fprintf(stderr, "[FATAL] failed to allocate memory\n"); + exit(EXIT_FAILURE); + } + if (rb_ec_raised_p(ec, RAISED_NOMEMORY)) { + rb_ec_raised_clear(ec); + } + else { + rb_ec_raised_set(ec, RAISED_NOMEMORY); + exc = ruby_vm_special_exception_copy(exc); + } + ec->errinfo = exc; + EC_JUMP_TAG(ec, TAG_RAISE); +} + +bool +rb_memerror_reentered(void) +{ + rb_execution_context_t *ec = GET_EC(); + return (ec && rb_ec_raised_p(ec, RAISED_NOMEMORY)); +} + +static void * +handle_malloc_failure(void *ptr) +{ + if (LIKELY(ptr)) { + return ptr; + } + else { + ruby_memerror(); + UNREACHABLE_RETURN(ptr); + } +} + +static void *ruby_xmalloc_body(size_t size); + +void * +ruby_xmalloc(size_t size) +{ + return handle_malloc_failure(ruby_xmalloc_body(size)); +} + +static bool +malloc_gc_allowed(void) +{ + rb_ractor_t *r = rb_current_ractor_raw(false); + + return r == NULL || !r->malloc_gc_disabled; +} + +static void * +ruby_xmalloc_body(size_t size) +{ + if ((ssize_t)size < 0) { + negative_size_allocation_error("too large allocation size"); + } + + return rb_gc_impl_malloc(rb_gc_get_objspace(), size, malloc_gc_allowed()); +} + +void +ruby_malloc_size_overflow(size_t count, size_t elsize) +{ + rb_raise(rb_eArgError, + "malloc: possible integer overflow (%"PRIuSIZE"*%"PRIuSIZE")", + count, elsize); +} + +void +ruby_malloc_add_size_overflow(size_t x, size_t y) +{ + rb_raise(rb_eArgError, + "malloc: possible integer overflow (%"PRIuSIZE"+%"PRIuSIZE")", + x, y); +} + +static void *ruby_xmalloc2_body(size_t n, size_t size); + +void * +ruby_xmalloc2(size_t n, size_t size) +{ + return handle_malloc_failure(ruby_xmalloc2_body(n, size)); +} + +static void * +ruby_xmalloc2_body(size_t n, size_t size) +{ + return rb_gc_impl_malloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed()); +} + +static void *ruby_xcalloc_body(size_t n, size_t size); + +void * +ruby_xcalloc(size_t n, size_t size) +{ + return handle_malloc_failure(ruby_xcalloc_body(n, size)); +} + +static void * +ruby_xcalloc_body(size_t n, size_t size) +{ + return rb_gc_impl_calloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed()); +} + +static void *ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size); + +#ifdef ruby_xrealloc_sized +#undef ruby_xrealloc_sized +#endif +void * +ruby_xrealloc_sized(void *ptr, size_t new_size, size_t old_size) +{ + return handle_malloc_failure(ruby_xrealloc_sized_body(ptr, new_size, old_size)); +} + +static void * +ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size) +{ + if ((ssize_t)new_size < 0) { + negative_size_allocation_error("too large allocation size"); + } + + return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, new_size, old_size, malloc_gc_allowed()); +} + +void * +ruby_xrealloc(void *ptr, size_t new_size) +{ + return ruby_xrealloc_sized(ptr, new_size, 0); +} + +static void *ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n); + +#ifdef ruby_xrealloc2_sized +#undef ruby_xrealloc2_sized +#endif +void * +ruby_xrealloc2_sized(void *ptr, size_t n, size_t size, size_t old_n) +{ + return handle_malloc_failure(ruby_xrealloc2_sized_body(ptr, n, size, old_n)); +} + +static void * +ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n) +{ + size_t len = xmalloc2_size(n, size); + return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, len, old_n * size, malloc_gc_allowed()); +} + +void * +ruby_xrealloc2(void *ptr, size_t n, size_t size) +{ + return ruby_xrealloc2_sized(ptr, n, size, 0); +} + +#ifdef ruby_xfree_sized +#undef ruby_xfree_sized +#endif +void +ruby_xfree_sized(void *x, size_t size) +{ + if (LIKELY(x)) { + /* It's possible for a C extension's pthread destructor function set by pthread_key_create + * to be called after ruby_vm_destruct and attempt to free memory. Fall back to mimfree in + * that case. */ + if (LIKELY(GET_VM())) { + rb_gc_impl_free(rb_gc_get_objspace(), x, size); + } + else { + ruby_mimfree(x); + } + } +} + +void +ruby_xfree(void *x) +{ + ruby_xfree_sized(x, 0); +} + +void * +rb_xmalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */ +{ + size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError); + return ruby_xmalloc(w); +} + +void * +rb_xcalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */ +{ + size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError); + return ruby_xcalloc(w, 1); +} + +void * +rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z) /* x * y + z */ +{ + size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError); + return ruby_xrealloc((void *)p, w); +} + +void * +rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */ +{ + size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError); + return ruby_xmalloc(u); +} + +void * +rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */ +{ + size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError); + return ruby_xcalloc(u, 1); +} + +/* Mimic ruby_xmalloc, but need not rb_objspace. + * should return pointer suitable for ruby_xfree + */ +void * +ruby_mimmalloc(size_t size) +{ + void *mem; +#if CALC_EXACT_MALLOC_SIZE + size += sizeof(struct malloc_obj_info); +#endif + mem = malloc(size); +#if CALC_EXACT_MALLOC_SIZE + if (!mem) { + return NULL; + } + else + /* set 0 for consistency of allocated_size/allocations */ + { + struct malloc_obj_info *info = mem; + info->size = 0; + mem = info + 1; + } +#endif + return mem; +} + +void * +ruby_mimcalloc(size_t num, size_t size) +{ + void *mem; +#if CALC_EXACT_MALLOC_SIZE + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(num, size); + if (UNLIKELY(t.overflowed)) { + return NULL; + } + size = t.result + sizeof(struct malloc_obj_info); + mem = calloc1(size); + if (!mem) { + return NULL; + } + else + /* set 0 for consistency of allocated_size/allocations */ + { + struct malloc_obj_info *info = mem; + info->size = 0; + mem = info + 1; + } +#else + mem = calloc(num, size); +#endif + return mem; +} + +void +ruby_mimfree(void *ptr) +{ +#if CALC_EXACT_MALLOC_SIZE + struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; + ptr = info; +#endif + free(ptr); +} + +void +rb_gc_adjust_memory_usage(ssize_t diff) +{ + unless_objspace(objspace) { return; } + + rb_gc_impl_adjust_memory_usage(objspace, diff); +} + +const char * +rb_obj_info(VALUE obj) +{ + return obj_info(obj); +} + +void +rb_obj_info_dump(VALUE obj) +{ + char buff[0x100]; + fprintf(stderr, "rb_obj_info_dump: %s\n", rb_raw_obj_info(buff, 0x100, obj)); +} + +void +rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func) +{ + char buff[0x100]; + fprintf(stderr, "<OBJ_INFO:%s@%s:%d> %s\n", func, file, line, rb_raw_obj_info(buff, 0x100, obj)); +} + +void +rb_gc_before_fork(void) { - unsigned long ptr, p0; + rb_gc_impl_before_fork(rb_gc_get_objspace()); +} + +void +rb_gc_after_fork(rb_pid_t pid) +{ + rb_gc_impl_after_fork(rb_gc_get_objspace(), pid); +} + +bool +rb_gc_obj_shareable_p(VALUE obj) +{ + return RB_OBJ_SHAREABLE_P(obj); +} - rb_secure(4); - p0 = ptr = NUM2ULONG(id); - if (ptr == Qtrue) return Qtrue; - if (ptr == Qfalse) return Qfalse; - if (ptr == Qnil) return Qnil; - if (FIXNUM_P(ptr)) return (VALUE)ptr; - if (SYMBOL_P(ptr) && rb_id2name(SYM2ID((VALUE)ptr)) != 0) { - return (VALUE)ptr; +void +rb_gc_rp(VALUE obj) +{ + rp(obj); +} + +struct check_shareable_data { + VALUE parent; + long err_count; +}; + +static void +check_shareable_i(const VALUE child, void *ptr) +{ + struct check_shareable_data *data = (struct check_shareable_data *)ptr; + + if (!rb_gc_obj_shareable_p(child)) { + fprintf(stderr, "(a) "); + rb_gc_rp(data->parent); + fprintf(stderr, "(b) "); + rb_gc_rp(child); + fprintf(stderr, "check_shareable_i: shareable (a) -> unshareable (b)\n"); + + data->err_count++; + rb_bug("!! violate shareable constraint !!"); } +} + +static bool gc_checking_shareable = false; - ptr = id ^ FIXNUM_FLAG; /* unset FIXNUM_FLAG */ - if (!is_pointer_to_heap((void *)ptr)|| BUILTIN_TYPE(ptr) >= T_BLKTAG) { - rb_raise(rb_eRangeError, "0x%lx is not id value", p0); +static void +gc_verify_shareable(void *objspace, VALUE obj, void *data) +{ + // while gc_checking_shareable is true, + // other Ractors should not run the GC, until the flag is not local. + // TODO: remove VM locking if the flag is Ractor local + + unsigned int lev = RB_GC_VM_LOCK(); + { + gc_checking_shareable = true; + rb_objspace_reachable_objects_from(obj, check_shareable_i, (void *)data); + gc_checking_shareable = false; } - if (BUILTIN_TYPE(ptr) == 0 || RBASIC(ptr)->klass == 0) { - rb_raise(rb_eRangeError, "0x%lx is recycled object", p0); + RB_GC_VM_UNLOCK(lev); +} + +// TODO: only one level (non-recursive) +void +rb_gc_verify_shareable(VALUE obj) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + struct check_shareable_data data = { + .parent = obj, + .err_count = 0, + }; + gc_verify_shareable(objspace, obj, &data); + + if (data.err_count > 0) { + rb_bug("rb_gc_verify_shareable"); } - return (VALUE)ptr; } +bool +rb_gc_checking_shareable(void) +{ + return gc_checking_shareable; +} + +/* + * Document-module: ObjectSpace + * + * The ObjectSpace module contains a number of routines + * that interact with the garbage collection facility and allow you to + * traverse all living objects with an iterator. + * + * ObjectSpace also provides support for object finalizers, procs that will be + * called after a specific object was destroyed by garbage collection. See + * the documentation for +ObjectSpace.define_finalizer+ for important + * information on how to use this method correctly. + * + * a = "A" + * b = "B" + * + * ObjectSpace.define_finalizer(a, proc {|id| puts "Finalizer one on #{id}" }) + * ObjectSpace.define_finalizer(b, proc {|id| puts "Finalizer two on #{id}" }) + * + * a = nil + * b = nil + * + * _produces:_ + * + * Finalizer two on 537763470 + * Finalizer one on 537763480 + */ + +/* Document-class: GC::Profiler + * + * The GC profiler provides access to information on GC runs including time, + * length and object space size. + * + * Example: + * + * GC::Profiler.enable + * + * require 'rdoc/rdoc' + * + * GC::Profiler.report + * + * GC::Profiler.disable + * + * See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations + */ + +#include "gc.rbinc" + void -Init_GC() +Init_GC(void) { - VALUE rb_mObSpace; +#undef rb_intern + rb_gc_register_address(&id2ref_value); + + malloc_offset = gc_compute_malloc_offset(); rb_mGC = rb_define_module("GC"); - rb_define_singleton_method(rb_mGC, "start", rb_gc_start, 0); - rb_define_singleton_method(rb_mGC, "enable", rb_gc_enable, 0); - rb_define_singleton_method(rb_mGC, "disable", rb_gc_disable, 0); - rb_define_method(rb_mGC, "garbage_collect", rb_gc_start, 0); - rb_mObSpace = rb_define_module("ObjectSpace"); - rb_define_module_function(rb_mObSpace, "each_object", os_each_obj, -1); - rb_define_module_function(rb_mObSpace, "garbage_collect", rb_gc_start, 0); - rb_define_module_function(rb_mObSpace, "add_finalizer", add_final, 1); - rb_define_module_function(rb_mObSpace, "remove_finalizer", rm_final, 1); - rb_define_module_function(rb_mObSpace, "finalizers", finals, 0); - rb_define_module_function(rb_mObSpace, "call_finalizer", call_final, 1); + VALUE rb_mObjSpace = rb_define_module("ObjectSpace"); - rb_define_module_function(rb_mObSpace, "define_finalizer", define_final, -1); - rb_define_module_function(rb_mObSpace, "undefine_finalizer", undefine_final, 1); + rb_define_module_function(rb_mObjSpace, "each_object", os_each_obj, -1); - rb_define_module_function(rb_mObSpace, "_id2ref", id2ref, 1); + rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1); + rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1); - rb_gc_register_address(&rb_mObSpace); - rb_global_variable(&finalizers); - rb_gc_unregister_address(&rb_mObSpace); - finalizers = rb_ary_new(); + rb_define_module_function(rb_mObjSpace, "_id2ref", os_id2ref, 1); - source_filenames = st_init_strtable(); + rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory"); - nomem_error = rb_exc_new2(rb_eNoMemError, "failed to allocate memory"); - rb_global_variable(&nomem_error); + rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0); + rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0); + + rb_define_module_function(rb_mObjSpace, "count_objects", count_objects, -1); + + rb_gc_impl_init(); +} + +// Set a name for the anonymous virtual memory area. `addr` is the starting +// address of the area and `size` is its length in bytes. `name` is a +// NUL-terminated human-readable string. +// +// This function is usually called after calling `mmap()`. The human-readable +// annotation helps developers identify the call site of `mmap()` that created +// the memory mapping. +// +// This function currently only works on Linux 5.17 or higher. After calling +// this function, we can see annotations in the form of "[anon:...]" in +// `/proc/self/maps`, where `...` is the content of `name`. This function has +// no effect when called on other platforms. +void +ruby_annotate_mmap(const void *addr, unsigned long size, const char *name) +{ +#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME) + // The name length cannot exceed 80 (including the '\0'). + RUBY_ASSERT(strlen(name) < 80); + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name); + // We ignore errors in prctl. prctl may set errno to EINVAL for several + // reasons. + // 1. The attr (PR_SET_VMA_ANON_NAME) is not a valid attribute. + // 2. addr is an invalid address. + // 3. The string pointed by name is too long. + // The first error indicates PR_SET_VMA_ANON_NAME is not available, and may + // happen if we run the compiled binary on an old kernel. In theory, all + // other errors should result in a failure. But since EINVAL cannot tell + // the first error from others, and this function is mainly used for + // debugging, we silently ignore the error. + errno = 0; +#endif } |