diff options
Diffstat (limited to 'gc')
31 files changed, 18320 insertions, 0 deletions
diff --git a/gc/README.md b/gc/README.md new file mode 100644 index 0000000000..cb71357973 --- /dev/null +++ b/gc/README.md @@ -0,0 +1,37 @@ +# Ruby's Garbage Collectors + +This directory contains implementations for Ruby's garbage collector (GC). The GC implementations use the Modular GC API to interact with Ruby. For more details about this API, see the [Modular GC API](#modular-gc-api) section. + +Two GC implementations are included in Ruby: + +- Default: The GC implementation that is used by default in Ruby. This GC is stable and production ready. The implementation uses a mark-sweep-compact algorithm. +- MMTk: An experimental implementation using the [MMTk](https://www.mmtk.io/) framework. The code lives in the [ruby/mmtk](https://github.com/ruby/mmtk) repository and is synchronized here. MMTk provides a [wide variety of GC algorithms](https://www.mmtk.io/status#implemented-collectors) to choose from. For usage instructions and current progress, refer to the [ruby/mmtk](https://github.com/ruby/mmtk) repository. + +## Building guide + +> [!TIP] +> If you are not sure how to build Ruby, follow the [Building Ruby](https://docs.ruby-lang.org/en/master/contributing/building_ruby_md.html) guide. + +> [!IMPORTANT] +> Ruby's modular GC feature is experimental and subject to change. There may be bugs or performance impacts. Use at your own risk. + +### Building Ruby with Modular GC + +1. Configure Ruby with the `--with-modular-gc=<dir>` option, where `dir` is the directory you want to place the built GC libraries into. +2. Build Ruby as usual. + +### Building GC implementations shipped with Ruby + +1. Build your desired GC implementation with `make install-modular-gc MODULAR_GC=<impl>`. This will build the GC implementation and place the built library into the `dir` specified in step 1. `impl` can be one of: + - `default`: The default GC that Ruby ships with. + - `mmtk`: The GC that uses [MMTk](https://www.mmtk.io/) as the back-end. See Ruby-specific details in the [ruby/mmtk](https://github.com/ruby/mmtk) repository. +2. Run your desired GC implementation by setting the `RUBY_GC_LIBRARY=<lib>` environment variable, where `lib` could be `default`, `mmtk`, or your own implementation (as long as you place it in the `dir` specified in step 1). + +## Modular GC API + +> [!WARNING] +> The Modular GC API is experimental and subject to change without notice. + +GC implementations interact with Ruby via the Modular GC API. All implementations must provide the functions in [gc/gc_impl.h](https://github.com/ruby/ruby/blob/master/gc/gc_impl.h) for Ruby to hook into. GC implementations can use any public C API in Ruby, along with additional APIs defined in [gc/gc.h](https://github.com/ruby/ruby/blob/master/gc/gc.h). + +Additionally, create an extconf.rb file to build the GC library. This file must use [gc/extconf_base.rb](https://github.com/ruby/ruby/blob/master/gc/extconf_base.rb) and the `create_gc_makefile` method. diff --git a/gc/default/default.c b/gc/default/default.c new file mode 100644 index 0000000000..0027f7a13c --- /dev/null +++ b/gc/default/default.c @@ -0,0 +1,9888 @@ +#include "ruby/internal/config.h" + +#include <signal.h> + +#ifndef _WIN32 +# include <sys/mman.h> +# include <unistd.h> +# ifdef HAVE_SYS_PRCTL_H +# include <sys/prctl.h> +# endif +#endif + +#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H) +/* LIST_HEAD conflicts with sys/queue.h on macOS */ +# include <sys/user.h> +#endif + +#ifdef BUILDING_MODULAR_GC +# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x)) +#else +# include "internal/bits.h" +#endif + +#include "ruby/ruby.h" +#include "ruby/atomic.h" +#include "ruby_atomic.h" +#include "ruby/debug.h" +#include "ruby/thread.h" +#include "ruby/util.h" +#include "ruby/vm.h" +#include "ruby/internal/encoding/string.h" +#include "ccan/list/list.h" +#include "darray.h" +#include "gc/gc.h" +#include "gc/gc_impl.h" + +#ifndef BUILDING_MODULAR_GC +# include "probes.h" +#endif + +#ifdef BUILDING_MODULAR_GC +# define RB_DEBUG_COUNTER_INC(_name) ((void)0) +# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond)) +#else +# include "debug_counter.h" +#endif + +#ifdef BUILDING_MODULAR_GC +# define rb_asan_poison_object(obj) ((void)(obj)) +# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p)) +# define asan_unpoisoning_object(obj) if ((obj) || true) +# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size)) +# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p)) +# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true) + +# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size)) +# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size)) +#else +# include "internal/sanitizers.h" +#endif + +/* MALLOC_HEADERS_BEGIN */ +#ifndef HAVE_MALLOC_USABLE_SIZE +# ifdef _WIN32 +# define HAVE_MALLOC_USABLE_SIZE +# define malloc_usable_size(a) _msize(a) +# elif defined HAVE_MALLOC_SIZE +# define HAVE_MALLOC_USABLE_SIZE +# define malloc_usable_size(a) malloc_size(a) +# endif +#endif + +#ifdef HAVE_MALLOC_USABLE_SIZE +# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER +/* Alternative malloc header is included in ruby/missing.h */ +# elif defined(HAVE_MALLOC_H) +# include <malloc.h> +# elif defined(HAVE_MALLOC_NP_H) +# include <malloc_np.h> +# elif defined(HAVE_MALLOC_MALLOC_H) +# include <malloc/malloc.h> +# endif +#endif + +#ifdef HAVE_MALLOC_TRIM +# include <malloc.h> + +# ifdef __EMSCRIPTEN__ +/* malloc_trim is defined in emscripten/emmalloc.h on emscripten. */ +# include <emscripten/emmalloc.h> +# endif +#endif + +#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS +# include <mach/task.h> +# include <mach/mach_init.h> +# include <mach/mach_port.h> +#endif + +#ifndef RUBY_DEBUG_LOG +# define RUBY_DEBUG_LOG(...) +#endif + +#ifndef GC_HEAP_INIT_BYTES +#define GC_HEAP_INIT_BYTES (2560 * 1024) +#endif +#ifndef GC_HEAP_FREE_SLOTS +#define GC_HEAP_FREE_SLOTS 4096 +#endif +#ifndef GC_HEAP_GROWTH_FACTOR +#define GC_HEAP_GROWTH_FACTOR 1.8 +#endif +#ifndef GC_HEAP_GROWTH_MAX_BYTES +#define GC_HEAP_GROWTH_MAX_BYTES 0 /* 0 is disable */ +#endif +#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO +# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01 +#endif +#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR +#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0 +#endif + +#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO +#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20 +#endif +#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO +#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40 +#endif +#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO +#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65 +#endif + +#ifndef GC_MALLOC_LIMIT_MIN +#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */) +#endif +#ifndef GC_MALLOC_LIMIT_MAX +#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */) +#endif +#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR +#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4 +#endif + +#ifndef GC_OLDMALLOC_LIMIT_MIN +#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */) +#endif +#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR +#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2 +#endif +#ifndef GC_OLDMALLOC_LIMIT_MAX +#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */) +#endif + +#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD +#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 /* 8KB */) +#endif + +#ifdef RB_THREAD_LOCAL_SPECIFIER +#define USE_MALLOC_INCREASE_LOCAL 1 +static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local; +#else +#define USE_MALLOC_INCREASE_LOCAL 0 +#endif + +#ifndef GC_CAN_COMPILE_COMPACTION +#if defined(__wasi__) /* WebAssembly doesn't support signals */ +# define GC_CAN_COMPILE_COMPACTION 0 +#else +# define GC_CAN_COMPILE_COMPACTION 1 +#endif +#endif + +#ifndef PRINT_ENTER_EXIT_TICK +# define PRINT_ENTER_EXIT_TICK 0 +#endif +#ifndef PRINT_ROOT_TICKS +#define PRINT_ROOT_TICKS 0 +#endif + +#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) + +#ifndef HEAP_COUNT +# if SIZEOF_VALUE >= 8 +# define HEAP_COUNT 12 +# else +# define HEAP_COUNT 5 +# endif +#endif + +/* The reciprocal table and pool_slot_sizes array are both generated from this + * single definition, so they can never get out of sync. */ +#if SIZEOF_VALUE >= 8 +# define EACH_POOL_SLOT_SIZE(SLOT) \ + SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \ + SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024) +#else +# define EACH_POOL_SLOT_SIZE(SLOT) \ + SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512) +#endif + +typedef struct ractor_newobj_heap_cache { + struct free_slot *freelist; + struct heap_page *using_page; + size_t allocated_objects_count; +} rb_ractor_newobj_heap_cache_t; + +typedef struct ractor_newobj_cache { + size_t incremental_mark_step_allocated_slots; + rb_ractor_newobj_heap_cache_t heap_caches[HEAP_COUNT]; +} rb_ractor_newobj_cache_t; + +typedef struct { + size_t heap_init_bytes; + size_t heap_free_slots; + double growth_factor; + size_t growth_max_bytes; + + double heap_free_slots_min_ratio; + double heap_free_slots_goal_ratio; + double heap_free_slots_max_ratio; + double uncollectible_wb_unprotected_objects_limit_ratio; + double oldobject_limit_factor; + + size_t malloc_limit_min; + size_t malloc_limit_max; + double malloc_limit_growth_factor; + + size_t oldmalloc_limit_min; + size_t oldmalloc_limit_max; + double oldmalloc_limit_growth_factor; +} ruby_gc_params_t; + +static ruby_gc_params_t gc_params = { + GC_HEAP_INIT_BYTES, + GC_HEAP_FREE_SLOTS, + GC_HEAP_GROWTH_FACTOR, + GC_HEAP_GROWTH_MAX_BYTES, + + GC_HEAP_FREE_SLOTS_MIN_RATIO, + GC_HEAP_FREE_SLOTS_GOAL_RATIO, + GC_HEAP_FREE_SLOTS_MAX_RATIO, + GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO, + GC_HEAP_OLDOBJECT_LIMIT_FACTOR, + + GC_MALLOC_LIMIT_MIN, + GC_MALLOC_LIMIT_MAX, + GC_MALLOC_LIMIT_GROWTH_FACTOR, + + GC_OLDMALLOC_LIMIT_MIN, + GC_OLDMALLOC_LIMIT_MAX, + GC_OLDMALLOC_LIMIT_GROWTH_FACTOR, +}; + +/* GC_DEBUG: + * enable to embed GC debugging information. + */ +#ifndef GC_DEBUG +#define GC_DEBUG 0 +#endif + +/* RGENGC_DEBUG: + * 1: basic information + * 2: remember set operation + * 3: mark + * 4: + * 5: sweep + */ +#ifndef RGENGC_DEBUG +#ifdef RUBY_DEVEL +#define RGENGC_DEBUG -1 +#else +#define RGENGC_DEBUG 0 +#endif +#endif +#if RGENGC_DEBUG < 0 && !defined(_MSC_VER) +# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level)) +#elif defined(HAVE_VA_ARGS_MACRO) +# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level)) +#else +# define RGENGC_DEBUG_ENABLED(level) 0 +#endif +int ruby_rgengc_debug; + +/* RGENGC_PROFILE + * 0: disable RGenGC profiling + * 1: enable profiling for basic information + * 2: enable profiling for each types + */ +#ifndef RGENGC_PROFILE +# define RGENGC_PROFILE 0 +#endif + +/* RGENGC_ESTIMATE_OLDMALLOC + * Enable/disable to estimate increase size of malloc'ed size by old objects. + * If estimation exceeds threshold, then will invoke full GC. + * 0: disable estimation. + * 1: enable estimation. + */ +#ifndef RGENGC_ESTIMATE_OLDMALLOC +# define RGENGC_ESTIMATE_OLDMALLOC 1 +#endif + +#ifndef GC_PROFILE_MORE_DETAIL +# define GC_PROFILE_MORE_DETAIL 0 +#endif +#ifndef GC_PROFILE_DETAIL_MEMORY +# define GC_PROFILE_DETAIL_MEMORY 0 +#endif +#ifndef GC_ENABLE_LAZY_SWEEP +# define GC_ENABLE_LAZY_SWEEP 1 +#endif + +#ifndef VERIFY_FREE_SIZE +#if RUBY_DEBUG +#define VERIFY_FREE_SIZE 1 +#else +#define VERIFY_FREE_SIZE 0 +#endif +#endif + +#if VERIFY_FREE_SIZE +#undef CALC_EXACT_MALLOC_SIZE +#define CALC_EXACT_MALLOC_SIZE 1 +#endif + +#ifndef CALC_EXACT_MALLOC_SIZE +# define CALC_EXACT_MALLOC_SIZE 0 +#endif + +#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0 +# ifndef MALLOC_ALLOCATED_SIZE +# define MALLOC_ALLOCATED_SIZE 0 +# endif +#else +# define MALLOC_ALLOCATED_SIZE 0 +#endif +#ifndef MALLOC_ALLOCATED_SIZE_CHECK +# define MALLOC_ALLOCATED_SIZE_CHECK 0 +#endif + +#ifndef GC_DEBUG_STRESS_TO_CLASS +# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG +#endif + +typedef enum { + GPR_FLAG_NONE = 0x000, + /* major reason */ + GPR_FLAG_MAJOR_BY_NOFREE = 0x001, + GPR_FLAG_MAJOR_BY_OLDGEN = 0x002, + GPR_FLAG_MAJOR_BY_SHADY = 0x004, + GPR_FLAG_MAJOR_BY_FORCE = 0x008, +#if RGENGC_ESTIMATE_OLDMALLOC + GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020, +#endif + GPR_FLAG_MAJOR_MASK = 0x0ff, + + /* gc reason */ + GPR_FLAG_NEWOBJ = 0x100, + GPR_FLAG_MALLOC = 0x200, + GPR_FLAG_METHOD = 0x400, + GPR_FLAG_CAPI = 0x800, + GPR_FLAG_STRESS = 0x1000, + + /* others */ + GPR_FLAG_IMMEDIATE_SWEEP = 0x2000, + GPR_FLAG_HAVE_FINALIZE = 0x4000, + GPR_FLAG_IMMEDIATE_MARK = 0x8000, + GPR_FLAG_FULL_MARK = 0x10000, + GPR_FLAG_COMPACT = 0x20000, + + GPR_DEFAULT_REASON = + (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | + GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI), +} gc_profile_record_flag; + +typedef struct gc_profile_record { + unsigned int flags; + + double gc_time; + double gc_invoke_time; + + size_t heap_total_objects; + size_t heap_use_size; + size_t heap_total_size; + size_t moved_objects; + +#if GC_PROFILE_MORE_DETAIL + double gc_mark_time; + double gc_sweep_time; + + size_t heap_use_pages; + size_t heap_live_objects; + size_t heap_free_objects; + + size_t allocate_increase; + size_t allocate_limit; + + double prepare_time; + size_t removing_objects; + size_t empty_objects; +#if GC_PROFILE_DETAIL_MEMORY + long maxrss; + long minflt; + long majflt; +#endif +#endif +#if MALLOC_ALLOCATED_SIZE + size_t allocated_size; +#endif + +#if RGENGC_PROFILE > 0 + size_t old_objects; + size_t remembered_normal_objects; + size_t remembered_shady_objects; +#endif +} gc_profile_record; + +struct RMoved { + VALUE flags; + VALUE dummy; + VALUE destination; +}; + +#define RMOVED(obj) ((struct RMoved *)(obj)) + +typedef uintptr_t bits_t; +enum { + BITS_SIZE = sizeof(bits_t), + BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT ) +}; + +struct heap_page_header { + struct heap_page *page; +}; + +struct heap_page_body { + struct heap_page_header header; + /* char gap[]; */ + /* RVALUE values[]; */ +}; + +#define STACK_CHUNK_SIZE 500 + +typedef struct stack_chunk { + VALUE data[STACK_CHUNK_SIZE]; + struct stack_chunk *next; +} stack_chunk_t; + +typedef struct mark_stack { + stack_chunk_t *chunk; + stack_chunk_t *cache; + int index; + int limit; + size_t cache_size; + size_t unused_cache_size; +} mark_stack_t; + +typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d); + +typedef struct rb_heap_struct { + short slot_size; + + /* Basic statistics */ + size_t total_allocated_pages; + size_t force_major_gc_count; + size_t force_incremental_marking_finish_count; + size_t total_allocated_objects; + size_t total_freed_objects; + size_t final_slots_count; + + /* Sweeping statistics */ + size_t freed_slots; + size_t empty_slots; + + struct heap_page *free_pages; + struct ccan_list_head pages; + struct heap_page *sweeping_page; /* iterator for .pages */ + struct heap_page *compact_cursor; + uintptr_t compact_cursor_index; + struct heap_page *pooled_pages; + size_t total_pages; /* total page count in a heap */ + size_t total_slots; /* total slot count */ + +} rb_heap_t; + +enum { + gc_stress_no_major, + gc_stress_no_immediate_sweep, + gc_stress_full_mark_after_malloc, + gc_stress_max +}; + +enum gc_mode { + gc_mode_none, + gc_mode_marking, + gc_mode_sweeping, + gc_mode_compacting, +}; + +typedef rbimpl_atomic_uint64_t gc_counter_t; + +#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \ + !(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))) +# define MALLOC_COUNTERS_NEED_LOCK 1 +#endif + +struct gc_malloc_bytes { + gc_counter_t malloc; + gc_counter_t free; + + /* Snapshots of `malloc` / `free` taken at the end of the last GC */ + gc_counter_t malloc_at_last_gc; + gc_counter_t free_at_last_gc; +}; + +typedef struct rb_objspace { + struct { + struct gc_malloc_bytes counters; +#if RGENGC_ESTIMATE_OLDMALLOC + struct gc_malloc_bytes oldcounters; +#endif +#ifdef MALLOC_COUNTERS_NEED_LOCK + rb_nativethread_lock_t lock; +#endif + } malloc_counters; + + struct { + size_t limit; +#if MALLOC_ALLOCATED_SIZE + size_t allocated_size; + size_t allocations; +#endif + } malloc_params; + + struct rb_gc_config { + bool full_mark; + } gc_config; + + struct { + unsigned int mode : 2; + unsigned int immediate_sweep : 1; + unsigned int dont_gc : 1; + unsigned int dont_incremental : 1; + unsigned int during_gc : 1; + unsigned int during_compacting : 1; + unsigned int during_reference_updating : 1; + unsigned int gc_stressful: 1; + unsigned int during_minor_gc : 1; + unsigned int during_incremental_marking : 1; + unsigned int measure_gc : 1; + } flags; + + rb_event_flag_t hook_events; + + rb_heap_t heaps[HEAP_COUNT]; + size_t empty_pages_count; + struct heap_page *empty_pages; + + struct { + rb_atomic_t finalizing; + } atomic_flags; + + mark_stack_t mark_stack; + size_t marked_slots; + + struct { + rb_darray(struct heap_page *) sorted; + + size_t allocated_pages; + size_t freed_pages; + uintptr_t range[2]; + size_t freeable_pages; + + size_t allocatable_bytes; + + /* final */ + VALUE deferred_final; + } heap_pages; + + st_table *finalizer_table; + + struct { + int run; + unsigned int latest_gc_info; + gc_profile_record *records; + gc_profile_record *current_record; + size_t next_index; + size_t size; + +#if GC_PROFILE_MORE_DETAIL + double prepare_time; +#endif + double invoke_time; + + size_t minor_gc_count; + size_t major_gc_count; + size_t compact_count; + size_t read_barrier_faults; +#if RGENGC_PROFILE > 0 + size_t total_generated_normal_object_count; + size_t total_generated_shady_object_count; + size_t total_shade_operation_count; + size_t total_promoted_count; + size_t total_remembered_normal_object_count; + size_t total_remembered_shady_object_count; + +#if RGENGC_PROFILE >= 2 + size_t generated_normal_object_count_types[RUBY_T_MASK]; + size_t generated_shady_object_count_types[RUBY_T_MASK]; + size_t shade_operation_count_types[RUBY_T_MASK]; + size_t promoted_types[RUBY_T_MASK]; + size_t remembered_normal_object_count_types[RUBY_T_MASK]; + size_t remembered_shady_object_count_types[RUBY_T_MASK]; +#endif +#endif /* RGENGC_PROFILE */ + + /* temporary profiling space */ + double gc_sweep_start_time; + size_t total_allocated_objects_at_gc_start; + size_t heap_used_at_gc_start; + size_t heap_total_slots_at_gc_start; + + /* basic statistics */ + size_t count; + unsigned long long marking_time_ns; + struct timespec marking_start_time; + unsigned long long sweeping_time_ns; + struct timespec sweeping_start_time; + + /* Weak references */ + size_t weak_references_count; + } profile; + + VALUE gc_stress_mode; + + struct { + bool parent_object_old_p; + VALUE parent_object; + + int need_major_gc; + size_t last_major_gc; + size_t uncollectible_wb_unprotected_objects; + size_t uncollectible_wb_unprotected_objects_limit; + size_t old_objects; + size_t old_objects_limit; + +#if RGENGC_ESTIMATE_OLDMALLOC + size_t oldmalloc_increase_limit; +#endif + +#if RGENGC_CHECK_MODE >= 2 + struct st_table *allrefs_table; + size_t error_count; +#endif + } rgengc; + + struct { + size_t considered_count_table[T_MASK]; + size_t moved_count_table[T_MASK]; + size_t moved_up_count_table[T_MASK]; + size_t moved_down_count_table[T_MASK]; + size_t total_moved; + + /* This function will be used, if set, to sort the heap prior to compaction */ + gc_compact_compare_func compare_func; + } rcompactor; + + struct { + size_t pooled_slots; + size_t step_slots; + } rincgc; + +#if GC_DEBUG_STRESS_TO_CLASS + VALUE stress_to_class; +#endif + + rb_darray(VALUE) weak_references; + rb_postponed_job_handle_t finalize_deferred_pjob; + + unsigned long live_ractor_cache_count; + + int sweeping_heap_count; + + int fork_vm_lock_lev; + + struct rb_gc_vm_context vm_context; +} rb_objspace_t; + +#ifndef HEAP_PAGE_ALIGN_LOG +/* default tiny heap size: 64KiB */ +#define HEAP_PAGE_ALIGN_LOG 16 +#endif + +#if RB_GC_OBJ_HAS_SUFFIX || GC_DEBUG +struct rvalue_overhead { +# if RB_GC_OBJ_HAS_SUFFIX + struct rb_gc_obj_suffix suffix; +# endif +# if GC_DEBUG + const char *file; + int line; +# endif +}; + +// Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE) +# define RVALUE_OVERHEAD (sizeof(struct { \ + union { \ + struct rvalue_overhead overhead; \ + VALUE value; \ + }; \ +})) +size_t rb_gc_impl_obj_slot_size(VALUE obj); +# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj))) +#else +# ifndef RVALUE_OVERHEAD +# define RVALUE_OVERHEAD 0 +# endif +#endif + +#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD) + +static const size_t pool_slot_sizes[HEAP_COUNT] = { +#define SLOT(size) ((size) + RVALUE_OVERHEAD), + EACH_POOL_SLOT_SIZE(SLOT) +#undef SLOT +}; + +/* Precomputed reciprocals for fast slot index calculation. + * For slot size d: reciprocal = ceil(2^48 / d). + * Then offset / d == (uint32_t)((offset * reciprocal) >> 48) + * for all offset < HEAP_PAGE_SIZE. */ +#define SLOT_RECIPROCAL_SHIFT 48 +#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size)) + +static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = { +#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD), + EACH_POOL_SLOT_SIZE(SLOT) +#undef SLOT +}; + +#if SIZEOF_VALUE >= 8 +static uint8_t size_to_heap_idx[1024 / 8 + 1]; +#else +static uint8_t size_to_heap_idx[512 / 8 + 1]; +#endif + +#ifndef MAX +# define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif +#ifndef MIN +# define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif +#define roomof(x, y) (((x) + (y) - 1) / (y)) +#define CEILDIV(i, mod) roomof(i, mod) +#define MIN_POOL_SLOT_SIZE 32 +enum { + HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG), + HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)), + HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN, + HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH), + HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT), +}; +#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG) +#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN + +#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS) +# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500 +#endif + +#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP +/* Must define either HEAP_PAGE_ALLOC_USE_MMAP or + * INIT_HEAP_PAGE_ALLOC_USE_MMAP. */ + +#ifndef HAVE_MMAP +/* We can't use mmap of course, if it is not available. */ +static const bool HEAP_PAGE_ALLOC_USE_MMAP = false; + +#elif defined(__wasm__) +/* wasmtime does not have proper support for mmap. + * See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends + */ +static const bool HEAP_PAGE_ALLOC_USE_MMAP = false; + +#elif HAVE_CONST_PAGE_SIZE +/* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */ +static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE); + +#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE) +/* If we can use the maximum page size. */ +static const bool HEAP_PAGE_ALLOC_USE_MMAP = true; + +#elif defined(PAGE_SIZE) +/* If the PAGE_SIZE macro can be used dynamically. */ +# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE) + +#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) +/* If we can use sysconf to determine the page size. */ +# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE) + +#else +/* Otherwise we can't determine the system page size, so don't use mmap. */ +static const bool HEAP_PAGE_ALLOC_USE_MMAP = false; +#endif + +#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP +/* We can determine the system page size at runtime. */ +# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false) + +static bool heap_page_alloc_use_mmap; +#endif + +#define RVALUE_AGE_BIT_COUNT 2 +#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1) +#define RVALUE_OLD_AGE 3 + +struct free_slot { + VALUE flags; /* always 0 for freed obj */ + struct free_slot *next; +}; + +struct heap_page { + /* Cache line 0: allocation fast path + SLOT_INDEX */ + struct free_slot *freelist; + uintptr_t start; + uint64_t slot_size_reciprocal; + unsigned short slot_size; + unsigned short total_slots; + unsigned short free_slots; + unsigned short final_slots; + unsigned short pinned_slots; + struct { + unsigned int before_sweep : 1; + unsigned int has_remembered_objects : 1; + unsigned int has_uncollectible_wb_unprotected_objects : 1; + } flags; + + rb_heap_t *heap; + + struct heap_page *free_next; + struct heap_page_body *body; + struct ccan_list_node page_node; + + bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT]; + /* the following three bitmaps are cleared at the beginning of full GC */ + bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT]; + bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT]; + bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT]; + + bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT]; + + /* If set, the object is not movable */ + bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT]; + bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT]; +}; + +/* + * When asan is enabled, this will prohibit writing to the freelist until it is unlocked + */ +static void +asan_lock_freelist(struct heap_page *page) +{ + asan_poison_memory_region(&page->freelist, sizeof(struct free_list *)); +} + +/* + * When asan is enabled, this will enable the ability to write to the freelist + */ +static void +asan_unlock_freelist(struct heap_page *page) +{ + asan_unpoison_memory_region(&page->freelist, sizeof(struct free_list *), false); +} + +static inline bool +heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *page) +{ + if (page->total_slots == 0) { + GC_ASSERT(page->start == 0); + GC_ASSERT(page->slot_size == 0); + GC_ASSERT(page->heap == NULL); + GC_ASSERT(page->free_slots == 0); + asan_unpoisoning_memory_region(&page->freelist, sizeof(&page->freelist)) { + GC_ASSERT(page->freelist == NULL); + } + + return true; + } + else { + GC_ASSERT(page->start != 0); + GC_ASSERT(page->slot_size != 0); + GC_ASSERT(page->heap != NULL); + + return false; + } +} + +#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK))) +#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header) +#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page) + +static inline size_t +slot_index_for_offset(size_t offset, uint64_t reciprocal) +{ + return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT); +} + +#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal) +#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH) +#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1)) +#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p)) + +#define _MARKED_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p)) +#define _MARK_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p)) +#define _CLEAR_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p)) + +#define MARKED_IN_BITMAP(bits, p) _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p) +#define MARK_IN_BITMAP(bits, p) _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p) +#define CLEAR_IN_BITMAP(bits, p) _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p) + +#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0]) +#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0]) +#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0]) +#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0]) +#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0]) + +static int +RVALUE_AGE_GET(VALUE obj) +{ + struct heap_page *page = GET_HEAP_PAGE(obj); + bits_t *age_bits = page->age_bits; + size_t slot_idx = SLOT_INDEX(page, obj); + size_t idx = (slot_idx / BITS_BITLENGTH) * 2; + int shift = (int)(slot_idx & (BITS_BITLENGTH - 1)); + int lo = (age_bits[idx] >> shift) & 1; + int hi = (age_bits[idx + 1] >> shift) & 1; + return lo | (hi << 1); +} + +static void +RVALUE_AGE_SET_BITMAP(VALUE obj, int age) +{ + RUBY_ASSERT(age <= RVALUE_OLD_AGE); + struct heap_page *page = GET_HEAP_PAGE(obj); + bits_t *age_bits = page->age_bits; + size_t slot_idx = SLOT_INDEX(page, obj); + size_t idx = (slot_idx / BITS_BITLENGTH) * 2; + int shift = (int)(slot_idx & (BITS_BITLENGTH - 1)); + bits_t mask = (bits_t)1 << shift; + + age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift); + age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift); +} + +static void +RVALUE_AGE_SET(VALUE obj, int age) +{ + RVALUE_AGE_SET_BITMAP(obj, age); + if (age == RVALUE_OLD_AGE) { + RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED); + } + else { + RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED); + } +} + +#define malloc_limit objspace->malloc_params.limit +#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters) +#define malloc_allocated_size objspace->malloc_params.allocated_size + +#ifdef MALLOC_COUNTERS_NEED_LOCK +# define MALLOC_COUNTERS_LOCK(o) rb_native_mutex_lock(&(o)->malloc_counters.lock) +# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock) +#else +# define MALLOC_COUNTERS_LOCK(o) ((void)0) +# define MALLOC_COUNTERS_UNLOCK(o) ((void)0) +#endif + +static inline void +gc_counter_add(gc_counter_t *p, size_t delta) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + *p += (gc_counter_t)delta; +#else + rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta); +#endif +} + +static inline gc_counter_t +gc_counter_load_relaxed(const gc_counter_t *p) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + return *p; +#else + return rbimpl_atomic_u64_load_relaxed(p); +#endif +} + +static inline gc_counter_t +gc_counter_load_acquire(const gc_counter_t *p) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + return *p; +#else + return rbimpl_atomic_u64_load_acquire(p); +#endif +} + +static inline void +gc_counter_store_release(gc_counter_t *p, gc_counter_t v) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + *p = v; +#else + rbimpl_atomic_u64_set_release(p, v); +#endif +} + +static inline int64_t +gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c) +{ + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc); + gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc); + gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc); + gc_counter_t free_now = gc_counter_load_relaxed(&c->free); + MALLOC_COUNTERS_UNLOCK(objspace); + + gc_counter_t malloc_delta = malloc_now - malloc_at; + gc_counter_t free_delta = free_now - free_at; + + if (malloc_delta >= free_delta) { + return (int64_t)(malloc_delta - free_delta); + } + else { + return -(int64_t)(free_delta - malloc_delta); + } +} + +static inline size_t +gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c) +{ + int64_t inc = gc_malloc_counters_increase(objspace, c); + if (inc <= 0) return 0; +#if SIZEOF_SIZE_T < 8 + if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX; +#endif + return (size_t)inc; +} + +static inline int64_t +gc_malloc_counters_snapshot(rb_objspace_t *objspace, struct gc_malloc_bytes *c) +{ + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc); + gc_counter_t free_now = gc_counter_load_relaxed(&c->free); + gc_counter_t malloc_at = gc_counter_load_relaxed(&c->malloc_at_last_gc); + gc_counter_t free_at = gc_counter_load_relaxed(&c->free_at_last_gc); + gc_counter_store_release(&c->malloc_at_last_gc, malloc_now); + gc_counter_store_release(&c->free_at_last_gc, free_now); + MALLOC_COUNTERS_UNLOCK(objspace); + + gc_counter_t malloc_delta = malloc_now - malloc_at; + gc_counter_t free_delta = free_now - free_at; + + if (malloc_delta >= free_delta) { + return (int64_t)(malloc_delta - free_delta); + } + else { + return -(int64_t)(free_delta - malloc_delta); + } +} + +#define heap_pages_lomem objspace->heap_pages.range[0] +#define heap_pages_himem objspace->heap_pages.range[1] +#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages +#define heap_pages_deferred_final objspace->heap_pages.deferred_final +#define heaps objspace->heaps +#define during_gc objspace->flags.during_gc +#define finalizing objspace->atomic_flags.finalizing +#define finalizer_table objspace->finalizer_table +#define ruby_gc_stressful objspace->flags.gc_stressful +#define ruby_gc_stress_mode objspace->gc_stress_mode +#if GC_DEBUG_STRESS_TO_CLASS +#define stress_to_class objspace->stress_to_class +#define set_stress_to_class(c) (stress_to_class = (c)) +#else +#define stress_to_class ((void)objspace, 0) +#define set_stress_to_class(c) ((void)objspace, (c)) +#endif + +#if 0 +#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1) +#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0) +#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b)) +#define dont_gc_val() (objspace->flags.dont_gc) +#else +#define dont_gc_on() (objspace->flags.dont_gc = 1) +#define dont_gc_off() (objspace->flags.dont_gc = 0) +#define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b)) +#define dont_gc_val() (objspace->flags.dont_gc) +#endif + +#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b)) +#define gc_config_full_mark_val (objspace->gc_config.full_mark) + +#ifndef DURING_GC_COULD_MALLOC_REGION_START +# define DURING_GC_COULD_MALLOC_REGION_START() \ + assert(rb_during_gc()); \ + bool _prev_enabled = rb_gc_impl_gc_enabled_p(objspace); \ + rb_gc_impl_gc_disable(objspace, false) +#endif + +#ifndef DURING_GC_COULD_MALLOC_REGION_END +# define DURING_GC_COULD_MALLOC_REGION_END() \ + if (_prev_enabled) rb_gc_impl_gc_enable(objspace) +#endif + +static inline enum gc_mode +gc_mode_verify(enum gc_mode mode) +{ +#if RGENGC_CHECK_MODE > 0 + switch (mode) { + case gc_mode_none: + case gc_mode_marking: + case gc_mode_sweeping: + case gc_mode_compacting: + break; + default: + rb_bug("gc_mode_verify: unreachable (%d)", (int)mode); + } +#endif + return mode; +} + +static inline bool +has_sweeping_pages(rb_objspace_t *objspace) +{ + return objspace->sweeping_heap_count != 0; +} + +static inline size_t +heap_eden_total_pages(rb_objspace_t *objspace) +{ + size_t count = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + count += (&heaps[i])->total_pages; + } + return count; +} + +static inline size_t +total_allocated_objects(rb_objspace_t *objspace) +{ + size_t count = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + count += heap->total_allocated_objects; + } + return count; +} + +static inline size_t +total_freed_objects(rb_objspace_t *objspace) +{ + size_t count = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + count += heap->total_freed_objects; + } + return count; +} + +static inline size_t +total_final_slots_count(rb_objspace_t *objspace) +{ + size_t count = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + count += heap->final_slots_count; + } + return count; +} + +#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode) +#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m)) +#define gc_needs_major_flags objspace->rgengc.need_major_gc + +#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking) +#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping) +#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE) +#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE) +#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE) +/* + * Byte budget for incremental sweep steps. Each step sweeps at most + * this many bytes worth of slots before yielding. The effective slot + * count per step is GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size, + * so larger slot pools (which are less heavily used) naturally get + * fewer slots swept per step. + * + * Baseline: 2048 slots * RVALUE_SLOT_SIZE = 2048 * 40 = 81920 bytes, + * preserving the historical behavior for the smallest heap. + */ +#define GC_INCREMENTAL_SWEEP_BYTES (2048 * RVALUE_SLOT_SIZE) +#define GC_INCREMENTAL_SWEEP_POOL_BYTES (1024 * RVALUE_SLOT_SIZE) +#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace)) +/* In lazy sweeping or the previous incremental marking finished and did not yield a free page. */ +#define needs_continue_sweeping(objspace, heap) \ + ((heap)->free_pages == NULL && is_lazy_sweeping(objspace)) + +#if SIZEOF_LONG == SIZEOF_VOIDP +# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */ +#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP +# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \ + ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1)) +#else +# error not supported +#endif + +struct RZombie { + VALUE flags; + VALUE next; + void (*dfree)(void *); + void *data; +}; + +#define RZOMBIE(o) ((struct RZombie *)(o)) + +static bool ruby_enable_autocompact = false; +#if RGENGC_CHECK_MODE +static gc_compact_compare_func ruby_autocompact_compare_func; +#endif + +static void init_mark_stack(mark_stack_t *stack); +static int garbage_collect(rb_objspace_t *, unsigned int reason); + +static int gc_start(rb_objspace_t *objspace, unsigned int reason); +static void gc_rest(rb_objspace_t *objspace); + +enum gc_enter_event { + gc_enter_event_start, + gc_enter_event_continue, + gc_enter_event_rest, + gc_enter_event_finalizer, +}; + +static inline void gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev); +static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev); +static void gc_marking_enter(rb_objspace_t *objspace); +static void gc_marking_exit(rb_objspace_t *objspace); +static void gc_sweeping_enter(rb_objspace_t *objspace); +static void gc_sweeping_exit(rb_objspace_t *objspace); +static bool gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap); + +static void gc_sweep(rb_objspace_t *objspace); +static void gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap); +static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap); + +static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr); +static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr); +static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr); + +static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count); +NO_SANITIZE("memory", static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)); + +static void gc_verify_internal_consistency(void *objspace_ptr); + +static double getrusage_time(void); +static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason); +static inline void gc_prof_timer_start(rb_objspace_t *); +static inline void gc_prof_timer_stop(rb_objspace_t *); +static inline void gc_prof_mark_timer_start(rb_objspace_t *); +static inline void gc_prof_mark_timer_stop(rb_objspace_t *); +static inline void gc_prof_sweep_timer_start(rb_objspace_t *); +static inline void gc_prof_sweep_timer_stop(rb_objspace_t *); +static inline void gc_prof_set_malloc_info(rb_objspace_t *); +static inline void gc_prof_set_heap_info(rb_objspace_t *); + +#define gc_prof_record(objspace) (objspace)->profile.current_record +#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record) + +#ifdef HAVE_VA_ARGS_MACRO +# define gc_report(level, objspace, ...) \ + if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__) +#else +# define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body +#endif +PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4); + +static void gc_finalize_deferred(void *dmy); + +#if USE_TICK_T + +/* the following code is only for internal tuning. */ + +/* Source code to use RDTSC is quoted and modified from + * https://www.mcs.anl.gov/~kazutomo/rdtsc.html + * written by Kazutomo Yoshii <kazutomo@mcs.anl.gov> + */ + +#if defined(__GNUC__) && defined(__i386__) +typedef unsigned long long tick_t; +#define PRItick "llu" +static inline tick_t +tick(void) +{ + unsigned long long int x; + __asm__ __volatile__ ("rdtsc" : "=A" (x)); + return x; +} + +#elif defined(__GNUC__) && defined(__x86_64__) +typedef unsigned long long tick_t; +#define PRItick "llu" + +static __inline__ tick_t +tick(void) +{ + unsigned long hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32); +} + +#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__)) +typedef unsigned long long tick_t; +#define PRItick "llu" + +static __inline__ tick_t +tick(void) +{ + unsigned long long val = __builtin_ppc_get_timebase(); + return val; +} + +#elif defined(__POWERPC__) && defined(__APPLE__) +/* Implementation for macOS PPC by @nobu + * See: https://github.com/ruby/ruby/pull/5975#discussion_r890045558 + */ +typedef unsigned long long tick_t; +#define PRItick "llu" + +static __inline__ tick_t +tick(void) +{ + unsigned long int upper, lower, tmp; + # define mftbu(r) __asm__ volatile("mftbu %0" : "=r"(r)) + # define mftb(r) __asm__ volatile("mftb %0" : "=r"(r)) + do { + mftbu(upper); + mftb(lower); + mftbu(tmp); + } while (tmp != upper); + return ((tick_t)upper << 32) | lower; +} + +#elif defined(__aarch64__) && defined(__GNUC__) +typedef unsigned long tick_t; +#define PRItick "lu" + +static __inline__ tick_t +tick(void) +{ + unsigned long val; + __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val)); + return val; +} + + +#elif defined(_WIN32) && defined(_MSC_VER) +#include <intrin.h> +typedef unsigned __int64 tick_t; +#define PRItick "llu" + +static inline tick_t +tick(void) +{ + return __rdtsc(); +} + +#else /* use clock */ +typedef clock_t tick_t; +#define PRItick "llu" + +static inline tick_t +tick(void) +{ + return clock(); +} +#endif /* TSC */ +#else /* USE_TICK_T */ +#define MEASURE_LINE(expr) expr +#endif /* USE_TICK_T */ + +static inline VALUE check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj); + +#define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj)) +#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj)) +#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj)) +#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj)) +#define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj)) + +static inline int +RVALUE_MARKED(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + return RVALUE_MARKED_BITMAP(obj) != 0; +} + +static inline int +RVALUE_PINNED(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + return RVALUE_PINNED_BITMAP(obj) != 0; +} + +static inline int +RVALUE_WB_UNPROTECTED(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0; +} + +static inline int +RVALUE_MARKING(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + return RVALUE_MARKING_BITMAP(obj) != 0; +} + +static inline int +RVALUE_REMEMBERED(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0; +} + +static inline int +RVALUE_UNCOLLECTIBLE(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0; +} + +#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj)) +#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj)) +#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj)) + +static int rgengc_remember(rb_objspace_t *objspace, VALUE obj); +static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap); +static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap); + +static int +check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int terminate) +{ + int err = 0; + + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + if (SPECIAL_CONST_P(obj)) { + fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj); + err++; + } + else if (!is_pointer_to_heap(objspace, (void *)obj)) { + struct heap_page *empty_page = objspace->empty_pages; + while (empty_page) { + if ((uintptr_t)empty_page->body <= (uintptr_t)obj && + (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) { + GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, empty_page)); + fprintf(stderr, "check_rvalue_consistency: %p is in an empty page (%p).\n", + (void *)obj, (void *)empty_page); + err++; + goto skip; + } + } + fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj); + err++; + skip: + ; + } + else { + const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0; + const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0; + const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0; + const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0; + const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0; + const int age = RVALUE_AGE_GET((VALUE)obj); + + if (heap_page_in_global_empty_pages_pool(objspace, GET_HEAP_PAGE(obj))) { + fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj)); + err++; + } + if (BUILTIN_TYPE(obj) == T_NONE) { + fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj)); + err++; + } + if (BUILTIN_TYPE(obj) == T_ZOMBIE) { + fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj)); + err++; + } + + if (BUILTIN_TYPE(obj) != T_DATA) { + rb_obj_memsize_of((VALUE)obj); + } + + /* check generation + * + * OLD == age == 3 && old-bitmap && mark-bit (except incremental marking) + */ + if (age > 0 && wb_unprotected_bit) { + fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age); + err++; + } + + if (!is_marking(objspace) && uncollectible_bit && !mark_bit) { + fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj)); + err++; + } + + if (!is_full_marking(objspace)) { + if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) { + fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n", + rb_obj_info(obj), age); + err++; + } + if (remembered_bit && age != RVALUE_OLD_AGE) { + fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n", + rb_obj_info(obj), age); + err++; + } + } + + /* + * check coloring + * + * marking:false marking:true + * marked:false white *invalid* + * marked:true black grey + */ + if (is_incremental_marking(objspace) && marking_bit) { + if (!is_marking(objspace) && !mark_bit) { + fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj)); + err++; + } + } + } + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); + + if (err > 0 && terminate) { + rb_bug("check_rvalue_consistency_force: there is %d errors.", err); + } + return err; +} + +#if RGENGC_CHECK_MODE == 0 +static inline VALUE +check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj) +{ + return obj; +} +#else +static VALUE +check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj) +{ + check_rvalue_consistency_force(objspace, obj, TRUE); + return obj; +} +#endif + +static inline bool +gc_object_moved_p(rb_objspace_t *objspace, VALUE obj) +{ + + bool ret; + asan_unpoisoning_object(obj) { + ret = BUILTIN_TYPE(obj) == T_MOVED; + } + return ret; +} + +static inline int +RVALUE_OLD_P(rb_objspace_t *objspace, VALUE obj) +{ + GC_ASSERT(!RB_SPECIAL_CONST_P(obj)); + check_rvalue_consistency(objspace, obj); + // Because this will only ever be called on GC controlled objects, + // we can use the faster _RAW function here + return RB_OBJ_PROMOTED_RAW(obj); +} + +static inline void +RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) +{ + MARK_IN_BITMAP(&page->uncollectible_bits[0], obj); + objspace->rgengc.old_objects++; + +#if RGENGC_PROFILE >= 2 + objspace->profile.total_promoted_count++; + objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++; +#endif +} + +static inline void +RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj) +{ + RB_DEBUG_COUNTER_INC(obj_promote); + RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj); +} + +/* set age to age+1 */ +static inline void +RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj) +{ + int age = RVALUE_AGE_GET((VALUE)obj); + + if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) { + rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj)); + } + + age++; + RVALUE_AGE_SET(obj, age); + + if (age == RVALUE_OLD_AGE) { + RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj); + } + + check_rvalue_consistency(objspace, obj); +} + +static inline void +RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + GC_ASSERT(!RVALUE_OLD_P(objspace, obj)); + RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1); + check_rvalue_consistency(objspace, obj); +} + +static inline void +RVALUE_AGE_RESET(VALUE obj) +{ + RVALUE_AGE_SET(obj, 0); +} + +static inline void +RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj) +{ + check_rvalue_consistency(objspace, obj); + GC_ASSERT(RVALUE_OLD_P(objspace, obj)); + + if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(objspace, obj)) { + CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj); + } + + CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj); + RVALUE_AGE_RESET(obj); + + if (RVALUE_MARKED(objspace, obj)) { + objspace->rgengc.old_objects--; + } + + check_rvalue_consistency(objspace, obj); +} + +static inline int +RVALUE_BLACK_P(rb_objspace_t *objspace, VALUE obj) +{ + return RVALUE_MARKED(objspace, obj) && !RVALUE_MARKING(objspace, obj); +} + +static inline int +RVALUE_WHITE_P(rb_objspace_t *objspace, VALUE obj) +{ + return !RVALUE_MARKED(objspace, obj); +} + +bool +rb_gc_impl_gc_enabled_p(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + return !dont_gc_val(); +} + +void +rb_gc_impl_gc_enable(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + dont_gc_off(); +} + +void +rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (finish_current_gc) { + gc_rest(objspace); + } + + dont_gc_on(); +} + +/* + --------------------------- ObjectSpace ----------------------------- +*/ + +static inline void * +calloc1(size_t n) +{ + return calloc(1, n); +} + +void +rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) +{ + rb_objspace_t *objspace = objspace_ptr; + objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK; +} + +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + unsigned long long marking_time = objspace->profile.marking_time_ns; + unsigned long long sweeping_time = objspace->profile.sweeping_time_ns; + + return marking_time + sweeping_time; +} + +void +rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) +{ + rb_objspace_t *objspace = objspace_ptr; + + objspace->flags.measure_gc = RTEST(flag) ? TRUE : FALSE; +} + +bool +rb_gc_impl_get_measure_total_time(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + return objspace->flags.measure_gc; +} + +/* garbage objects will be collected soon. */ +bool +rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + bool dead = false; + + asan_unpoisoning_object(ptr) { + switch (BUILTIN_TYPE(ptr)) { + case T_NONE: + case T_MOVED: + case T_ZOMBIE: + dead = true; + break; + default: + break; + } + } + + if (dead) return true; + return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep && + !RVALUE_MARKED(objspace, ptr); +} + +struct rb_gc_vm_context * +rb_gc_impl_get_vm_context(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + return &objspace->vm_context; +} + +static void free_stack_chunks(mark_stack_t *); +static void mark_stack_free_cache(mark_stack_t *); +static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page); + +static inline void +heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) +{ + rb_asan_unpoison_object(obj, false); + + asan_unlock_freelist(page); + + struct free_slot *slot = (struct free_slot *)obj; + slot->flags = 0; + slot->next = page->freelist; + page->freelist = slot; + asan_lock_freelist(page); + + // Should have already been reset + GC_ASSERT(RVALUE_AGE_GET(obj) == 0); + + if (RGENGC_CHECK_MODE && + /* obj should belong to page */ + !(page->start <= (uintptr_t)obj && + (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) && + obj % sizeof(VALUE) == 0)) { + rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj); + } + + rb_asan_poison_object(obj); + gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj); +} + +static void +heap_allocatable_bytes_expand(rb_objspace_t *objspace, + rb_heap_t *heap, size_t free_slots, size_t total_slots, size_t slot_size) +{ + double goal_ratio = gc_params.heap_free_slots_goal_ratio; + size_t target_total_slots; + + if (goal_ratio == 0.0) { + target_total_slots = (size_t)(total_slots * gc_params.growth_factor); + } + else if (total_slots == 0) { + target_total_slots = gc_params.heap_init_bytes / slot_size; + } + else { + /* Find `f' where free_slots = f * total_slots * goal_ratio + * => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots) + */ + double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots); + + if (f > gc_params.growth_factor) f = gc_params.growth_factor; + if (f < 1.0) f = 1.1; + + target_total_slots = (size_t)(f * total_slots); + + if (0) { + fprintf(stderr, + "free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f," + " G(%1.2f), f(%1.2f)," + " total_slots(%8"PRIuSIZE") => target_total_slots(%8"PRIuSIZE")\n", + free_slots, total_slots, free_slots/(double)total_slots, + goal_ratio, f, total_slots, target_total_slots); + } + } + + if (gc_params.growth_max_bytes > 0) { + size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size; + if (target_total_slots > max_total_slots) target_total_slots = max_total_slots; + } + + size_t extend_slot_count = target_total_slots - total_slots; + /* Extend by at least 1 page. */ + if (extend_slot_count == 0) extend_slot_count = 1; + + objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size; +} + +static inline void +heap_add_freepage(rb_heap_t *heap, struct heap_page *page) +{ + asan_unlock_freelist(page); + GC_ASSERT(page->free_slots != 0); + GC_ASSERT(page->freelist != NULL); + + page->free_next = heap->free_pages; + heap->free_pages = page; + + RUBY_DEBUG_LOG("page:%p freelist:%p", (void *)page, (void *)page->freelist); + + asan_lock_freelist(page); +} + +static inline void +heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) +{ + asan_unlock_freelist(page); + GC_ASSERT(page->free_slots != 0); + GC_ASSERT(page->freelist != NULL); + + page->free_next = heap->pooled_pages; + heap->pooled_pages = page; + objspace->rincgc.pooled_slots += page->free_slots; + + asan_lock_freelist(page); +} + +static void +heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) +{ + ccan_list_del(&page->page_node); + heap->total_pages--; + heap->total_slots -= page->total_slots; +} + +static void +gc_aligned_free(void *ptr, size_t size) +{ +#if defined __MINGW32__ + __mingw_aligned_free(ptr); +#elif defined _WIN32 + _aligned_free(ptr); +#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN) + free(ptr); +#else + free(((void**)ptr)[-1]); +#endif +} + +static void +heap_page_body_free(struct heap_page_body *page_body) +{ + GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0); + + if (HEAP_PAGE_ALLOC_USE_MMAP) { +#ifdef HAVE_MMAP + GC_ASSERT(HEAP_PAGE_SIZE % sysconf(_SC_PAGE_SIZE) == 0); + if (munmap(page_body, HEAP_PAGE_SIZE)) { + rb_bug("heap_page_body_free: munmap failed"); + } +#endif + } + else { + gc_aligned_free(page_body, HEAP_PAGE_SIZE); + } +} + +static void +heap_page_free(rb_objspace_t *objspace, struct heap_page *page) +{ + objspace->heap_pages.freed_pages++; + heap_page_body_free(page->body); + free(page); +} + +static void +heap_pages_free_unused_pages(rb_objspace_t *objspace) +{ + if (objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) { + GC_ASSERT(objspace->empty_pages_count > 0); + objspace->empty_pages = NULL; + objspace->empty_pages_count = 0; + + size_t i, j; + for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { + struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); + + if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) { + heap_page_free(objspace, page); + heap_pages_freeable_pages--; + } + else { + if (heap_page_in_global_empty_pages_pool(objspace, page)) { + page->free_next = objspace->empty_pages; + objspace->empty_pages = page; + objspace->empty_pages_count++; + } + + if (i != j) { + rb_darray_set(objspace->heap_pages.sorted, j, page); + } + j++; + } + } + + rb_darray_pop(objspace->heap_pages.sorted, i - j); + GC_ASSERT(rb_darray_size(objspace->heap_pages.sorted) == j); + + struct heap_page *hipage = rb_darray_get(objspace->heap_pages.sorted, rb_darray_size(objspace->heap_pages.sorted) - 1); + uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE; + GC_ASSERT(himem <= heap_pages_himem); + heap_pages_himem = himem; + + struct heap_page *lopage = rb_darray_get(objspace->heap_pages.sorted, 0); + uintptr_t lomem = (uintptr_t)lopage->body + sizeof(struct heap_page_header); + GC_ASSERT(lomem >= heap_pages_lomem); + heap_pages_lomem = lomem; + } +} + +static void * +gc_aligned_malloc(size_t alignment, size_t size) +{ + /* alignment must be a power of 2 */ + GC_ASSERT(((alignment - 1) & alignment) == 0); + GC_ASSERT(alignment % sizeof(void*) == 0); + + void *res; + +#if defined __MINGW32__ + res = __mingw_aligned_malloc(size, alignment); +#elif defined _WIN32 + void *_aligned_malloc(size_t, size_t); + res = _aligned_malloc(size, alignment); +#elif defined(HAVE_POSIX_MEMALIGN) + if (posix_memalign(&res, alignment, size) != 0) { + return NULL; + } +#elif defined(HAVE_MEMALIGN) + res = memalign(alignment, size); +#else + char* aligned; + res = malloc(alignment + size + sizeof(void*)); + aligned = (char*)res + alignment + sizeof(void*); + aligned -= ((VALUE)aligned & (alignment - 1)); + ((void**)aligned)[-1] = res; + res = (void*)aligned; +#endif + + GC_ASSERT((uintptr_t)res % alignment == 0); + + return res; +} + +static struct heap_page_body * +heap_page_body_allocate(void) +{ + struct heap_page_body *page_body; + + if (HEAP_PAGE_ALLOC_USE_MMAP) { +#ifdef HAVE_MMAP + GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0); + + size_t mmap_size = HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE; + char *ptr = mmap(NULL, mmap_size, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) { + return NULL; + } + + // If we are building `default.c` as part of the ruby executable, we + // may just call `ruby_annotate_mmap`. But if we are building + // `default.c` as a shared library, we will not have access to private + // symbols, and we have to either call prctl directly or make our own + // wrapper. +#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME) + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size, "Ruby:GC:default:heap_page_body_allocate"); + errno = 0; +#endif + + char *aligned = ptr + HEAP_PAGE_ALIGN; + aligned -= ((VALUE)aligned & (HEAP_PAGE_ALIGN - 1)); + GC_ASSERT(aligned > ptr); + GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN); + + size_t start_out_of_range_size = aligned - ptr; + GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0); + if (start_out_of_range_size > 0) { + if (munmap(ptr, start_out_of_range_size)) { + rb_bug("heap_page_body_allocate: munmap failed for start"); + } + } + + size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size; + GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0); + if (end_out_of_range_size > 0) { + if (munmap(aligned + HEAP_PAGE_SIZE, end_out_of_range_size)) { + rb_bug("heap_page_body_allocate: munmap failed for end"); + } + } + + page_body = (struct heap_page_body *)aligned; +#endif + } + else { + page_body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE); + } + + GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0); + + return page_body; +} + +static struct heap_page * +heap_page_resurrect(rb_objspace_t *objspace) +{ + struct heap_page *page = NULL; + if (objspace->empty_pages == NULL) { + GC_ASSERT(objspace->empty_pages_count == 0); + } + else { + GC_ASSERT(objspace->empty_pages_count > 0); + objspace->empty_pages_count--; + page = objspace->empty_pages; + objspace->empty_pages = page->free_next; + } + + return page; +} + +static struct heap_page * +heap_page_allocate(rb_objspace_t *objspace) +{ + struct heap_page_body *page_body = heap_page_body_allocate(); + if (page_body == 0) { + rb_memerror(); + } + + struct heap_page *page = calloc1(sizeof(struct heap_page)); + if (page == 0) { + heap_page_body_free(page_body); + rb_memerror(); + } + + uintptr_t start = (uintptr_t)page_body + sizeof(struct heap_page_header); + uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE; + + size_t lo = 0; + size_t hi = rb_darray_size(objspace->heap_pages.sorted); + while (lo < hi) { + struct heap_page *mid_page; + + size_t mid = (lo + hi) / 2; + mid_page = rb_darray_get(objspace->heap_pages.sorted, mid); + if ((uintptr_t)mid_page->start < start) { + lo = mid + 1; + } + else if ((uintptr_t)mid_page->start > start) { + hi = mid; + } + else { + rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid); + } + } + + rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page); + + if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start; + if (heap_pages_himem < end) heap_pages_himem = end; + + page->body = page_body; + page_body->header.page = page; + + objspace->heap_pages.allocated_pages++; + + return page; +} + +static void +heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) +{ + /* Adding to eden heap during incremental sweeping is forbidden */ + GC_ASSERT(!heap->sweeping_page); + GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); + + /* Align start to slot_size boundary */ + uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header); + uintptr_t rem = start % heap->slot_size; + if (rem) start += heap->slot_size - rem; + + int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size); + + page->start = start; + page->total_slots = slot_count; + page->slot_size = heap->slot_size; + page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps]; + page->heap = heap; + + memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + memset(&page->age_bits[0], 0, sizeof(page->age_bits)); + + asan_unlock_freelist(page); + page->freelist = NULL; + asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false); + for (VALUE p = (VALUE)start; p < start + (slot_count * heap->slot_size); p += heap->slot_size) { + heap_page_add_freeobj(objspace, page, p); + } + asan_lock_freelist(page); + + page->free_slots = slot_count; + + heap->total_allocated_pages++; + + ccan_list_add_tail(&heap->pages, &page->page_node); + heap->total_pages++; + heap->total_slots += page->total_slots; +} + +static int +heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap) +{ + gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", " + "allocatable_bytes: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n", + rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_bytes, heap->total_pages); + + bool allocated = false; + struct heap_page *page = heap_page_resurrect(objspace); + + if (page == NULL && objspace->heap_pages.allocatable_bytes > 0) { + page = heap_page_allocate(objspace); + allocated = true; + + GC_ASSERT(page != NULL); + } + + if (page != NULL) { + heap_add_page(objspace, heap, page); + heap_add_freepage(heap, page); + + if (allocated) { + size_t page_bytes = (size_t)page->total_slots * page->slot_size; + if (objspace->heap_pages.allocatable_bytes > page_bytes) { + objspace->heap_pages.allocatable_bytes -= page_bytes; + } + else { + objspace->heap_pages.allocatable_bytes = 0; + } + } + } + + return page != NULL; +} + +static void +heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap) +{ + size_t prev_allocatable_bytes = objspace->heap_pages.allocatable_bytes; + objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE; + heap_page_allocate_and_initialize(objspace, heap); + GC_ASSERT(heap->free_pages != NULL); + objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes; +} + +static void +gc_continue(rb_objspace_t *objspace, rb_heap_t *heap) +{ + unsigned int lock_lev; + bool needs_gc = is_incremental_marking(objspace) || needs_continue_sweeping(objspace, heap); + if (!needs_gc) return; + + gc_enter(objspace, gc_enter_event_continue, &lock_lev); // takes vm barrier, try to avoid + + /* Continue marking if in incremental marking. */ + if (is_incremental_marking(objspace)) { + if (gc_marks_continue(objspace, heap)) { + gc_sweep(objspace); + } + } + + if (needs_continue_sweeping(objspace, heap)) { + gc_sweep_continue(objspace, heap); + } + + gc_exit(objspace, gc_enter_event_continue, &lock_lev); +} + +static void +heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap) +{ + GC_ASSERT(heap->free_pages == NULL); + + if (heap->total_slots < gc_params.heap_init_bytes / heap->slot_size && + heap->sweeping_page == NULL) { + heap_page_allocate_and_initialize_force(objspace, heap); + GC_ASSERT(heap->free_pages != NULL); + return; + } + + /* Continue incremental marking or lazy sweeping, if in any of those steps. */ + gc_continue(objspace, heap); + + if (heap->free_pages == NULL) { + heap_page_allocate_and_initialize(objspace, heap); + } + + /* If we still don't have a free page and not allowed to create a new page, + * we should start a new GC cycle. */ + if (heap->free_pages == NULL) { + GC_ASSERT(objspace->empty_pages_count == 0); + GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0); + + if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) { + rb_memerror(); + } + else { + if (objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) { + heap_allocatable_bytes_expand(objspace, heap, + heap->freed_slots + heap->empty_slots, + heap->total_slots, heap->slot_size); + GC_ASSERT(objspace->heap_pages.allocatable_bytes > 0); + } + /* Do steps of incremental marking or lazy sweeping if the GC run permits. */ + gc_continue(objspace, heap); + + /* If we're not incremental marking (e.g. a minor GC) or finished + * sweeping and still don't have a free page, then + * gc_sweep_finish_heap should allow us to create a new page. */ + if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, heap)) { + if (gc_needs_major_flags == GPR_FLAG_NONE) { + rb_bug("cannot create a new page after GC"); + } + else { // Major GC is required, which will allow us to create new page + if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) { + rb_memerror(); + } + else { + /* Do steps of incremental marking or lazy sweeping. */ + gc_continue(objspace, heap); + + if (heap->free_pages == NULL && + !heap_page_allocate_and_initialize(objspace, heap)) { + rb_bug("cannot create a new page after major GC"); + } + } + } + } + } + } + + GC_ASSERT(heap->free_pages != NULL); +} + +#if GC_DEBUG +static inline const char* +rb_gc_impl_source_location_cstr(int *ptr) +{ + /* We could directly refer `rb_source_location_cstr()` before, but not any + * longer. We have to heavy lift using our debugging API. */ + if (! ptr) { + return NULL; + } + else if (! (*ptr = rb_sourceline())) { + return NULL; + } + else { + return rb_sourcefile(); + } +} +#endif + +static inline VALUE +newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj) +{ + GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE); + GC_ASSERT((flags & FL_WB_PROTECTED) == 0); + RBASIC(obj)->flags = flags; + *((VALUE *)&RBASIC(obj)->klass) = klass; +#if RBASIC_SHAPE_ID_FIELD + RBASIC(obj)->shape_id = 0; +#endif + +#if RGENGC_CHECK_MODE + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + check_rvalue_consistency(objspace, obj); + + GC_ASSERT(RVALUE_MARKED(objspace, obj) == FALSE); + GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE); + GC_ASSERT(RVALUE_OLD_P(objspace, obj) == FALSE); + GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, obj) == FALSE); + + if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj)); + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +#endif + + if (RB_UNLIKELY(wb_protected == FALSE)) { + MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj); + } + +#if RGENGC_PROFILE + if (wb_protected) { + objspace->profile.total_generated_normal_object_count++; +#if RGENGC_PROFILE >= 2 + objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++; +#endif + } + else { + objspace->profile.total_generated_shady_object_count++; +#if RGENGC_PROFILE >= 2 + objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++; +#endif + } +#endif + +#if GC_DEBUG + GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line); + GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */ +#endif + + gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj)); + + // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj)); + return obj; +} + +size_t +rb_gc_impl_obj_slot_size(VALUE obj) +{ + return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD; +} + +static inline size_t +heap_slot_size(unsigned char pool_id) +{ + GC_ASSERT(pool_id < HEAP_COUNT); + + return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD; +} + +bool +rb_gc_impl_size_allocatable_p(size_t size) +{ + return size + RVALUE_OVERHEAD <= pool_slot_sizes[HEAP_COUNT - 1]; +} + +static const size_t ALLOCATED_COUNT_STEP = 1024; +static void +ractor_cache_flush_count(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache) +{ + for (int heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) { + rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx]; + + rb_heap_t *heap = &heaps[heap_idx]; + RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count); + heap_cache->allocated_objects_count = 0; + } +} + +static inline VALUE +ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, + size_t heap_idx) +{ + rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx]; + struct free_slot *p = heap_cache->freelist; + + if (RB_UNLIKELY(is_incremental_marking(objspace))) { + // Not allowed to allocate without running an incremental marking step + if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) { + return Qfalse; + } + + if (p) { + cache->incremental_mark_step_allocated_slots++; + } + } + + if (RB_LIKELY(p)) { + VALUE obj = (VALUE)p; + rb_asan_unpoison_object(obj, true); + heap_cache->freelist = p->next; + + heap_cache->allocated_objects_count++; + rb_heap_t *heap = &heaps[heap_idx]; + if (heap_cache->allocated_objects_count >= ALLOCATED_COUNT_STEP) { + RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count); + heap_cache->allocated_objects_count = 0; + } + +#if RGENGC_CHECK_MODE + GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx)); + // zero clear + MEMZERO((char *)obj, char, heap_slot_size(heap_idx)); +#endif + return obj; + } + else { + return Qfalse; + } +} + +static struct heap_page * +heap_next_free_page(rb_objspace_t *objspace, rb_heap_t *heap) +{ + struct heap_page *page; + + if (heap->free_pages == NULL) { + heap_prepare(objspace, heap); + } + + page = heap->free_pages; + heap->free_pages = page->free_next; + + GC_ASSERT(page->free_slots != 0); + + asan_unlock_freelist(page); + + return page; +} + +static inline void +ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, + struct heap_page *page) +{ + gc_report(3, objspace, "ractor_set_cache: Using page %p\n", (void *)page->body); + + rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx]; + + GC_ASSERT(heap_cache->freelist == NULL); + GC_ASSERT(page->free_slots != 0); + GC_ASSERT(page->freelist != NULL); + + heap_cache->using_page = page; + heap_cache->freelist = page->freelist; + page->free_slots = 0; + page->freelist = NULL; + + rb_asan_unpoison_object((VALUE)heap_cache->freelist, false); + GC_ASSERT(RB_TYPE_P((VALUE)heap_cache->freelist, T_NONE)); + rb_asan_poison_object((VALUE)heap_cache->freelist); +} + +static void +init_size_to_heap_idx(void) +{ + for (size_t i = 0; i < sizeof(size_to_heap_idx); i++) { + size_t effective = i * 8 + RVALUE_OVERHEAD; + uint8_t idx; + for (idx = 0; idx < HEAP_COUNT; idx++) { + if (effective <= pool_slot_sizes[idx]) break; + } + size_to_heap_idx[i] = idx; + } +} + +static inline size_t +heap_idx_for_size(size_t size) +{ + size_t compressed = (size + 7) >> 3; + if (compressed < sizeof(size_to_heap_idx)) { + size_t heap_idx = size_to_heap_idx[compressed]; + if (RB_LIKELY(heap_idx < HEAP_COUNT)) return heap_idx; + } + + rb_bug("heap_idx_for_size: allocation size too large " + "(size=%"PRIuSIZE")", size); +} + +size_t +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) +{ + return heap_idx_for_size(size); +} + + +static size_t heap_sizes[HEAP_COUNT + 1] = { 0 }; + +size_t * +rb_gc_impl_heap_sizes(void *objspace_ptr) +{ + if (heap_sizes[0] == 0) { + for (unsigned char i = 0; i < HEAP_COUNT; i++) { + heap_sizes[i] = heap_slot_size(i); + } + } + + return heap_sizes; +} + +NOINLINE(static VALUE newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked)); + +static VALUE +newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked) +{ + rb_heap_t *heap = &heaps[heap_idx]; + VALUE obj = Qfalse; + + unsigned int lev = 0; + bool unlock_vm = false; + + if (!vm_locked) { + lev = RB_GC_CR_LOCK(); + unlock_vm = true; + } + + { + if (is_incremental_marking(objspace)) { + gc_continue(objspace, heap); + cache->incremental_mark_step_allocated_slots = 0; + + // Retry allocation after resetting incremental_mark_step_allocated_slots + obj = ractor_cache_allocate_slot(objspace, cache, heap_idx); + } + + if (obj == Qfalse) { + // Get next free page (possibly running GC) + struct heap_page *page = heap_next_free_page(objspace, heap); + ractor_cache_set_page(objspace, cache, heap_idx, page); + + // Retry allocation after moving to new page + obj = ractor_cache_allocate_slot(objspace, cache, heap_idx); + } + } + + if (unlock_vm) { + RB_GC_CR_UNLOCK(lev); + } + + if (RB_UNLIKELY(obj == Qfalse)) { + rb_memerror(); + } + return obj; +} + +static VALUE +newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked) +{ + VALUE obj = ractor_cache_allocate_slot(objspace, cache, heap_idx); + + if (RB_UNLIKELY(obj == Qfalse)) { + obj = newobj_cache_miss(objspace, cache, heap_idx, vm_locked); + } + + return obj; +} + +ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t heap_idx)); + +static inline VALUE +newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t heap_idx) +{ + VALUE obj; + unsigned int lev; + + lev = RB_GC_CR_LOCK(); + { + if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) { + if (during_gc) { + dont_gc_on(); + during_gc = 0; + if (rb_memerror_reentered()) { + rb_memerror(); + } + rb_bug("object allocation during garbage collection phase"); + } + + if (ruby_gc_stressful) { + if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) { + rb_memerror(); + } + } + } + + obj = newobj_alloc(objspace, cache, heap_idx, true); + newobj_init(klass, flags, wb_protected, objspace, obj); + } + RB_GC_CR_UNLOCK(lev); + + return obj; +} + +NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, + rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx)); +NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, + rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx)); + +static VALUE +newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx) +{ + return newobj_slowpath(klass, flags, objspace, cache, TRUE, heap_idx); +} + +static VALUE +newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx) +{ + return newobj_slowpath(klass, flags, objspace, cache, FALSE, heap_idx); +} + +VALUE +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) +{ + VALUE obj; + rb_objspace_t *objspace = objspace_ptr; + + RB_DEBUG_COUNTER_INC(obj_newobj); + (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected); + + if (RB_UNLIKELY(stress_to_class)) { + if (rb_hash_lookup2(stress_to_class, klass, Qundef) != Qundef) { + rb_memerror(); + } + } + + size_t heap_idx = heap_idx_for_size(alloc_size); + + rb_ractor_newobj_cache_t *cache = (rb_ractor_newobj_cache_t *)cache_ptr; + + if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) && + wb_protected) { + obj = newobj_alloc(objspace, cache, heap_idx, false); + newobj_init(klass, flags, wb_protected, objspace, obj); + } + else { + RB_DEBUG_COUNTER_INC(obj_newobj_slowpath); + + obj = wb_protected ? + newobj_slowpath_wb_protected(klass, flags, objspace, cache, heap_idx) : + newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, heap_idx); + } + + return obj; +} + +static int +ptr_in_page_body_p(const void *ptr, const void *memb) +{ + struct heap_page *page = *(struct heap_page **)memb; + uintptr_t p_body = (uintptr_t)page->body; + + if ((uintptr_t)ptr >= p_body) { + return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1; + } + else { + return -1; + } +} + +PUREFUNC(static inline struct heap_page *heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);) +static inline struct heap_page * +heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr) +{ + struct heap_page **res; + + if (ptr < (uintptr_t)heap_pages_lomem || + ptr > (uintptr_t)heap_pages_himem) { + return NULL; + } + + res = bsearch((void *)ptr, rb_darray_ref(objspace->heap_pages.sorted, 0), + rb_darray_size(objspace->heap_pages.sorted), sizeof(struct heap_page *), + ptr_in_page_body_p); + + if (res) { + return *res; + } + else { + return NULL; + } +} + +PUREFUNC(static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr);) +static inline bool +is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr) +{ + register uintptr_t p = (uintptr_t)ptr; + register struct heap_page *page; + + RB_DEBUG_COUNTER_INC(gc_isptr_trial); + + if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE; + RB_DEBUG_COUNTER_INC(gc_isptr_range); + + if (p % sizeof(VALUE) != 0) return FALSE; + RB_DEBUG_COUNTER_INC(gc_isptr_align); + + page = heap_page_for_ptr(objspace, (uintptr_t)ptr); + if (page) { + RB_DEBUG_COUNTER_INC(gc_isptr_maybe); + if (heap_page_in_global_empty_pages_pool(objspace, page)) { + return FALSE; + } + else { + if (p < page->start) return FALSE; + if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE; + if ((p - page->start) % page->slot_size != 0) return FALSE; + + return TRUE; + } + } + return FALSE; +} + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + return is_pointer_to_heap(objspace_ptr, ptr); +} + +#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE) + +void +rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) +{ + rb_objspace_t *objspace = objspace_ptr; + + struct RZombie *zombie = RZOMBIE(obj); + zombie->flags = T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS); + zombie->dfree = dfree; + zombie->data = data; + VALUE prev, next = heap_pages_deferred_final; + do { + zombie->next = prev = next; + next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj); + } while (next != prev); + + struct heap_page *page = GET_HEAP_PAGE(obj); + page->final_slots++; + page->heap->final_slots_count++; +} + +typedef int each_obj_callback(void *, void *, size_t, void *); +typedef int each_page_callback(struct heap_page *, void *); + +struct each_obj_data { + rb_objspace_t *objspace; + bool reenable_incremental; + + each_obj_callback *each_obj_callback; + each_page_callback *each_page_callback; + void *data; + + struct heap_page **pages[HEAP_COUNT]; + size_t pages_counts[HEAP_COUNT]; +}; + +static VALUE +objspace_each_objects_ensure(VALUE arg) +{ + struct each_obj_data *data = (struct each_obj_data *)arg; + rb_objspace_t *objspace = data->objspace; + + /* Reenable incremental GC */ + if (data->reenable_incremental) { + objspace->flags.dont_incremental = FALSE; + } + + for (int i = 0; i < HEAP_COUNT; i++) { + struct heap_page **pages = data->pages[i]; + free(pages); + } + + return Qnil; +} + +static VALUE +objspace_each_objects_try(VALUE arg) +{ + struct each_obj_data *data = (struct each_obj_data *)arg; + rb_objspace_t *objspace = data->objspace; + + /* Copy pages from all heaps to their respective buffers. */ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + size_t size = heap->total_pages * sizeof(struct heap_page *); + + struct heap_page **pages = malloc(size); + if (!pages) rb_memerror(); + + /* Set up pages buffer by iterating over all pages in the current eden + * heap. This will be a snapshot of the state of the heap before we + * call the callback over each page that exists in this buffer. Thus it + * is safe for the callback to allocate objects without possibly entering + * an infinite loop. */ + struct heap_page *page = 0; + size_t pages_count = 0; + ccan_list_for_each(&heap->pages, page, page_node) { + pages[pages_count] = page; + pages_count++; + } + data->pages[i] = pages; + data->pages_counts[i] = pages_count; + GC_ASSERT(pages_count == heap->total_pages); + } + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + size_t pages_count = data->pages_counts[i]; + struct heap_page **pages = data->pages[i]; + + struct heap_page *page = ccan_list_top(&heap->pages, struct heap_page, page_node); + for (size_t i = 0; i < pages_count; i++) { + /* If we have reached the end of the linked list then there are no + * more pages, so break. */ + if (page == NULL) break; + + /* If this page does not match the one in the buffer, then move to + * the next page in the buffer. */ + if (pages[i] != page) continue; + + uintptr_t pstart = (uintptr_t)page->start; + uintptr_t pend = pstart + (page->total_slots * heap->slot_size); + + if (data->each_obj_callback && + (*data->each_obj_callback)((void *)pstart, (void *)pend, heap->slot_size, data->data)) { + break; + } + if (data->each_page_callback && + (*data->each_page_callback)(page, data->data)) { + break; + } + + page = ccan_list_next(&heap->pages, page, page_node); + } + } + + return Qnil; +} + +static void +objspace_each_exec(bool protected, struct each_obj_data *each_obj_data) +{ + /* Disable incremental GC */ + rb_objspace_t *objspace = each_obj_data->objspace; + bool reenable_incremental = FALSE; + if (protected) { + reenable_incremental = !objspace->flags.dont_incremental; + + gc_rest(objspace); + objspace->flags.dont_incremental = TRUE; + } + + each_obj_data->reenable_incremental = reenable_incremental; + memset(&each_obj_data->pages, 0, sizeof(each_obj_data->pages)); + memset(&each_obj_data->pages_counts, 0, sizeof(each_obj_data->pages_counts)); + rb_ensure(objspace_each_objects_try, (VALUE)each_obj_data, + objspace_each_objects_ensure, (VALUE)each_obj_data); +} + +static void +objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected) +{ + struct each_obj_data each_obj_data = { + .objspace = objspace, + .each_obj_callback = callback, + .each_page_callback = NULL, + .data = data, + }; + objspace_each_exec(protected, &each_obj_data); +} + +void +rb_gc_impl_each_objects(void *objspace_ptr, each_obj_callback *callback, void *data) +{ + objspace_each_objects(objspace_ptr, callback, data, TRUE); +} + +#if GC_CAN_COMPILE_COMPACTION +static void +objspace_each_pages(rb_objspace_t *objspace, each_page_callback *callback, void *data, bool protected) +{ + struct each_obj_data each_obj_data = { + .objspace = objspace, + .each_obj_callback = NULL, + .each_page_callback = callback, + .data = data, + }; + objspace_each_exec(protected, &each_obj_data); +} +#endif + +VALUE +rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) +{ + rb_objspace_t *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + GC_ASSERT(!OBJ_FROZEN(obj)); + + RBASIC(obj)->flags |= FL_FINALIZE; + + unsigned int lev = RB_GC_VM_LOCK(); + + if (st_lookup(finalizer_table, obj, &data)) { + table = (VALUE)data; + VALUE dup_table = rb_ary_dup(table); + + RB_GC_VM_UNLOCK(lev); + /* avoid duplicate block, table is usually small */ + { + long len = RARRAY_LEN(table); + long i; + + for (i = 0; i < len; i++) { + VALUE recv = RARRAY_AREF(dup_table, i); + if (rb_equal(recv, block)) { // can't be called with VM lock held + return recv; + } + } + } + lev = RB_GC_VM_LOCK(); + RB_GC_GUARD(dup_table); + + rb_ary_push(table, block); + } + else { + table = rb_ary_new3(2, rb_obj_id(obj), block); + rb_obj_hide(table); + st_add_direct(finalizer_table, obj, table); + } + + RB_GC_VM_UNLOCK(lev); + + return block; +} + +void +rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + GC_ASSERT(!OBJ_FROZEN(obj)); + + st_data_t data = obj; + + int lev = RB_GC_VM_LOCK(); + st_delete(finalizer_table, &data, 0); + RB_GC_VM_UNLOCK(lev); + + FL_UNSET(obj, FL_FINALIZE); +} + +void +rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + if (!FL_TEST(obj, FL_FINALIZE)) return; + + int lev = RB_GC_VM_LOCK(); + if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) { + table = rb_ary_dup((VALUE)data); + RARRAY_ASET(table, 0, rb_obj_id(dest)); + st_insert(finalizer_table, dest, table); + FL_SET(dest, FL_FINALIZE); + } + else { + rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj)); + } + RB_GC_VM_UNLOCK(lev); +} + +static VALUE +get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i + 1); +} + +static unsigned int +run_final(rb_objspace_t *objspace, VALUE zombie, unsigned int lev) +{ + if (RZOMBIE(zombie)->dfree) { + RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data); + } + + st_data_t key = (st_data_t)zombie; + if (FL_TEST_RAW(zombie, FL_FINALIZE)) { + FL_UNSET(zombie, FL_FINALIZE); + st_data_t table; + if (st_delete(finalizer_table, &key, &table)) { + RB_GC_VM_UNLOCK(lev); + rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table); + lev = RB_GC_VM_LOCK(); + } + else { + rb_bug("FL_FINALIZE flag is set, but finalizers are not found"); + } + } + else { + GC_ASSERT(!st_lookup(finalizer_table, key, NULL)); + } + return lev; +} + +static void +finalize_list(rb_objspace_t *objspace, VALUE zombie) +{ + while (zombie) { + VALUE next_zombie; + struct heap_page *page; + rb_asan_unpoison_object(zombie, false); + next_zombie = RZOMBIE(zombie)->next; + page = GET_HEAP_PAGE(zombie); + + unsigned int lev = RB_GC_VM_LOCK(); + + lev = run_final(objspace, zombie, lev); + { + GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE); + GC_ASSERT(page->heap->final_slots_count > 0); + GC_ASSERT(page->final_slots > 0); + + page->heap->final_slots_count--; + page->final_slots--; + page->free_slots++; + RVALUE_AGE_SET_BITMAP(zombie, 0); + heap_page_add_freeobj(objspace, page, zombie); + page->heap->total_freed_objects++; + } + RB_GC_VM_UNLOCK(lev); + + zombie = next_zombie; + } +} + +static void +finalize_deferred_heap_pages(rb_objspace_t *objspace) +{ + VALUE zombie; + while ((zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) { + finalize_list(objspace, zombie); + } +} + +static void +finalize_deferred(rb_objspace_t *objspace) +{ + rb_gc_set_pending_interrupt(); + finalize_deferred_heap_pages(objspace); + rb_gc_unset_pending_interrupt(); +} + +static void +gc_finalize_deferred(void *dmy) +{ + rb_objspace_t *objspace = dmy; + if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) return; + + finalize_deferred(objspace); + RUBY_ATOMIC_SET(finalizing, 0); +} + +static void +gc_finalize_deferred_register(rb_objspace_t *objspace) +{ + /* will enqueue a call to gc_finalize_deferred */ + rb_postponed_job_trigger(objspace->finalize_deferred_pjob); +} + +static int pop_mark_stack(mark_stack_t *stack, VALUE *data); + +static void +gc_abort(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (is_incremental_marking(objspace)) { + /* Remove all objects from the mark stack. */ + VALUE obj; + while (pop_mark_stack(&objspace->mark_stack, &obj)); + + objspace->flags.during_incremental_marking = FALSE; + } + + if (is_lazy_sweeping(objspace)) { + objspace->sweeping_heap_count = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + heap->sweeping_page = NULL; + struct heap_page *page = NULL; + + ccan_list_for_each(&heap->pages, page, page_node) { + page->flags.before_sweep = false; + } + } + } + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + rgengc_mark_and_rememberset_clear(objspace, heap); + } + + gc_mode_set(objspace, gc_mode_none); +} + +void +rb_gc_impl_shutdown_free_objects(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { + struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); + short stride = page->slot_size; + + uintptr_t p = (uintptr_t)page->start; + uintptr_t pend = p + page->total_slots * stride; + for (; p < pend; p += stride) { + VALUE vp = (VALUE)p; + asan_unpoisoning_object(vp) { + if (RB_BUILTIN_TYPE(vp) != T_NONE) { + rb_gc_obj_free_vm_weak_references(vp); + if (rb_gc_obj_free(objspace, vp)) { + RBASIC(vp)->flags = 0; + } + } + } + } + } +} + +static int +rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data) +{ + VALUE obj = (VALUE)key; + VALUE table = (VALUE)val; + + GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE)); + GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY); + + rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table); + + FL_UNSET(obj, FL_FINALIZE); + + return ST_DELETE; +} + +void +rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + +#if RGENGC_CHECK_MODE >= 2 + gc_verify_internal_consistency(objspace); +#endif + + /* prohibit incremental GC */ + objspace->flags.dont_incremental = 1; + + if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) { + /* Abort incremental marking and lazy sweeping to speed up shutdown. */ + gc_abort(objspace); + dont_gc_on(); + return; + } + + while (finalizer_table->num_entries) { + st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0); + } + + /* run finalizers */ + finalize_deferred(objspace); + GC_ASSERT(heap_pages_deferred_final == 0); + + /* Abort incremental marking and lazy sweeping to speed up shutdown. */ + gc_abort(objspace); + + /* prohibit GC because force T_DATA finalizers can break an object graph consistency */ + dont_gc_on(); + + /* running data/file finalizers are part of garbage collection */ + unsigned int lock_lev; + gc_enter(objspace, gc_enter_event_finalizer, &lock_lev); + + /* run data/file object's finalizers */ + for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { + struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); + short stride = page->slot_size; + + uintptr_t p = (uintptr_t)page->start; + uintptr_t pend = p + page->total_slots * stride; + for (; p < pend; p += stride) { + VALUE vp = (VALUE)p; + asan_unpoisoning_object(vp) { + if (rb_gc_shutdown_call_finalizer_p(vp)) { + rb_gc_obj_free_vm_weak_references(vp); + if (rb_gc_obj_free(objspace, vp)) { + RBASIC(vp)->flags = 0; + } + } + } + } + } + + gc_exit(objspace, gc_enter_event_finalizer, &lock_lev); + + finalize_deferred_heap_pages(objspace); + + st_free_table(finalizer_table); + finalizer_table = 0; + RUBY_ATOMIC_SET(finalizing, 0); +} + +void +rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data) +{ + rb_objspace_t *objspace = objspace_ptr; + + for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { + struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); + short stride = page->slot_size; + + uintptr_t p = (uintptr_t)page->start; + uintptr_t pend = p + page->total_slots * stride; + for (; p < pend; p += stride) { + VALUE obj = (VALUE)p; + + asan_unpoisoning_object(obj) { + func(obj, data); + } + } + } +} + +/* + ------------------------ Garbage Collection ------------------------ +*/ + +/* Sweeping */ + +static size_t +objspace_available_slots(rb_objspace_t *objspace) +{ + size_t total_slots = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + total_slots += heap->total_slots; + } + return total_slots; +} + +static size_t +objspace_live_slots(rb_objspace_t *objspace) +{ + return total_allocated_objects(objspace) - total_freed_objects(objspace) - total_final_slots_count(objspace); +} + +static size_t +objspace_free_slots(rb_objspace_t *objspace) +{ + return objspace_available_slots(objspace) - objspace_live_slots(objspace) - total_final_slots_count(objspace); +} + +static void +gc_setup_mark_bits(struct heap_page *page) +{ + /* copy oldgen bitmap to mark bitmap */ + memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE); +} + +static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj); +static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct heap_page *src_page, struct heap_page *dest_page); + +#if defined(_WIN32) +enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE}; + +static BOOL +protect_page_body(struct heap_page_body *body, DWORD protect) +{ + DWORD old_protect; + return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0; +} +#elif defined(__wasi__) +// wasi-libc's mprotect emulation does not support PROT_NONE +enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK}; +#define protect_page_body(body, protect) 1 +#else +enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE}; +#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect)) +#endif + +static void +lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body) +{ + if (!protect_page_body(body, HEAP_PAGE_LOCK)) { + rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno)); + } + else { + gc_report(5, objspace, "Protecting page in move %p\n", (void *)body); + } +} + +static void +unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body) +{ + if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) { + rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno)); + } + else { + gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body); + } +} + +static bool +try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src) +{ + GC_ASSERT(gc_is_moveable_obj(objspace, src)); + + struct heap_page *src_page = GET_HEAP_PAGE(src); + if (!free_page) { + return false; + } + + /* We should return true if either src is successfully moved, or src is + * unmoveable. A false return will cause the sweeping cursor to be + * incremented to the next page, and src will attempt to move again */ + GC_ASSERT(RVALUE_MARKED(objspace, src)); + + asan_unlock_freelist(free_page); + VALUE dest = (VALUE)free_page->freelist; + asan_lock_freelist(free_page); + if (dest) { + rb_asan_unpoison_object(dest, false); + } + else { + /* if we can't get something from the freelist then the page must be + * full */ + return false; + } + asan_unlock_freelist(free_page); + free_page->freelist = ((struct free_slot *)dest)->next; + asan_lock_freelist(free_page); + + GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE); + + if (src_page->slot_size > free_page->slot_size) { + objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++; + } + else if (free_page->slot_size > src_page->slot_size) { + objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++; + } + objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++; + objspace->rcompactor.total_moved++; + + gc_move(objspace, src, dest, src_page, free_page); + gc_pin(objspace, src); + free_page->free_slots--; + + return true; +} + +static void +gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap) +{ + struct heap_page *cursor = heap->compact_cursor; + + while (cursor) { + unlock_page_body(objspace, cursor->body); + cursor = ccan_list_next(&heap->pages, cursor, page_node); + } +} + +static void gc_update_references(rb_objspace_t *objspace); +#if GC_CAN_COMPILE_COMPACTION +static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page); +#endif + +#if defined(__MINGW32__) || defined(_WIN32) +# define GC_COMPACTION_SUPPORTED 1 +#else +/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for + * the read barrier, so we must disable compaction. */ +# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP) +#endif + +#if GC_CAN_COMPILE_COMPACTION +static void +read_barrier_handler(uintptr_t address) +{ + rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace(); + + struct heap_page_body *page_body = GET_PAGE_BODY(address); + + /* If the page_body is NULL, then mprotect cannot handle it and will crash + * with "Cannot allocate memory". */ + if (page_body == NULL) { + rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address); + } + + int lev = RB_GC_VM_LOCK(); + { + unlock_page_body(objspace, page_body); + + objspace->profile.read_barrier_faults++; + + invalidate_moved_page(objspace, GET_HEAP_PAGE(address)); + } + RB_GC_VM_UNLOCK(lev); +} +#endif + +#if !GC_CAN_COMPILE_COMPACTION +static void +uninstall_handlers(void) +{ + /* no-op */ +} + +static void +install_handlers(void) +{ + /* no-op */ +} +#elif defined(_WIN32) +static LPTOP_LEVEL_EXCEPTION_FILTER old_handler; +typedef void (*signal_handler)(int); +static signal_handler old_sigsegv_handler; + +static LONG WINAPI +read_barrier_signal(EXCEPTION_POINTERS *info) +{ + /* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */ + if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + /* > The second array element specifies the virtual address of the inaccessible data. + * https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record + * + * Use this address to invalidate the page */ + read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]); + return EXCEPTION_CONTINUE_EXECUTION; + } + else { + return EXCEPTION_CONTINUE_SEARCH; + } +} + +static void +uninstall_handlers(void) +{ + signal(SIGSEGV, old_sigsegv_handler); + SetUnhandledExceptionFilter(old_handler); +} + +static void +install_handlers(void) +{ + /* Remove SEGV handler so that the Unhandled Exception Filter handles it */ + old_sigsegv_handler = signal(SIGSEGV, NULL); + /* Unhandled Exception Filter has access to the violation address similar + * to si_addr from sigaction */ + old_handler = SetUnhandledExceptionFilter(read_barrier_signal); +} +#else +static struct sigaction old_sigbus_handler; +static struct sigaction old_sigsegv_handler; + +#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS +static exception_mask_t old_exception_masks[32]; +static mach_port_t old_exception_ports[32]; +static exception_behavior_t old_exception_behaviors[32]; +static thread_state_flavor_t old_exception_flavors[32]; +static mach_msg_type_number_t old_exception_count; + +static void +disable_mach_bad_access_exc(void) +{ + old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]); + task_swap_exception_ports( + mach_task_self(), EXC_MASK_BAD_ACCESS, + MACH_PORT_NULL, EXCEPTION_DEFAULT, 0, + old_exception_masks, &old_exception_count, + old_exception_ports, old_exception_behaviors, old_exception_flavors + ); +} + +static void +restore_mach_bad_access_exc(void) +{ + for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) { + task_set_exception_ports( + mach_task_self(), + old_exception_masks[i], old_exception_ports[i], + old_exception_behaviors[i], old_exception_flavors[i] + ); + } +} +#endif + +static void +read_barrier_signal(int sig, siginfo_t *info, void *data) +{ + // setup SEGV/BUS handlers for errors + struct sigaction prev_sigbus, prev_sigsegv; + sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus); + sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv); + + // enable SIGBUS/SEGV + sigset_t set, prev_set; + sigemptyset(&set); + sigaddset(&set, SIGBUS); + sigaddset(&set, SIGSEGV); + sigprocmask(SIG_UNBLOCK, &set, &prev_set); +#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS + disable_mach_bad_access_exc(); +#endif + // run handler + read_barrier_handler((uintptr_t)info->si_addr); + + // reset SEGV/BUS handlers +#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS + restore_mach_bad_access_exc(); +#endif + sigaction(SIGBUS, &prev_sigbus, NULL); + sigaction(SIGSEGV, &prev_sigsegv, NULL); + sigprocmask(SIG_SETMASK, &prev_set, NULL); +} + +static void +uninstall_handlers(void) +{ +#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS + restore_mach_bad_access_exc(); +#endif + sigaction(SIGBUS, &old_sigbus_handler, NULL); + sigaction(SIGSEGV, &old_sigsegv_handler, NULL); +} + +static void +install_handlers(void) +{ + struct sigaction action; + memset(&action, 0, sizeof(struct sigaction)); + sigemptyset(&action.sa_mask); + action.sa_sigaction = read_barrier_signal; + action.sa_flags = SA_SIGINFO | SA_ONSTACK; + + sigaction(SIGBUS, &action, &old_sigbus_handler); + sigaction(SIGSEGV, &action, &old_sigsegv_handler); +#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS + disable_mach_bad_access_exc(); +#endif +} +#endif + +static void +gc_compact_finish(rb_objspace_t *objspace) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + gc_unprotect_pages(objspace, heap); + } + + uninstall_handlers(); + + gc_update_references(objspace); + objspace->profile.compact_count++; + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + heap->compact_cursor = NULL; + heap->free_pages = NULL; + heap->compact_cursor_index = 0; + } + + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects; + } + objspace->flags.during_compacting = FALSE; +} + +struct gc_sweep_context { + struct heap_page *page; + int final_slots; + int freed_slots; + int empty_slots; +}; + +static inline void +gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx) +{ + struct heap_page *sweep_page = ctx->page; + short slot_size = sweep_page->slot_size; + + do { + VALUE vp = (VALUE)p; + GC_ASSERT(vp % sizeof(VALUE) == 0); + + rb_asan_unpoison_object(vp, false); + if (bitset & 1) { + switch (BUILTIN_TYPE(vp)) { + case T_MOVED: + if (objspace->flags.during_compacting) { + /* The sweep cursor shouldn't have made it to any + * T_MOVED slots while the compact flag is enabled. + * The sweep cursor and compact cursor move in + * opposite directions, and when they meet references will + * get updated and "during_compacting" should get disabled */ + rb_bug("T_MOVED shouldn't be seen until compaction is finished"); + } + gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); + ctx->empty_slots++; + heap_page_add_freeobj(objspace, sweep_page, vp); + break; + case T_ZOMBIE: + /* already counted */ + break; + case T_NONE: + ctx->empty_slots++; /* already freed */ + break; + + default: +#if RGENGC_CHECK_MODE + if (!is_full_marking(objspace)) { + if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p); + if (RVALUE_REMEMBERED(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p); + } +#endif + +#if RGENGC_CHECK_MODE +#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp)) + CHECK(RVALUE_WB_UNPROTECTED); + CHECK(RVALUE_MARKED); + CHECK(RVALUE_MARKING); + CHECK(RVALUE_UNCOLLECTIBLE); +#undef CHECK +#endif + + if (!rb_gc_obj_needs_cleanup_p(vp)) { + (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size); + heap_page_add_freeobj(objspace, sweep_page, vp); + gc_report(3, objspace, "page_sweep: %s (fast path) added to freelist\n", rb_obj_info(vp)); + ctx->freed_slots++; + } + else { + gc_report(2, objspace, "page_sweep: free %p\n", (void *)p); + + rb_gc_obj_free_vm_weak_references(vp); + if (rb_gc_obj_free(objspace, vp)) { + (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size); + heap_page_add_freeobj(objspace, sweep_page, vp); + gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); + ctx->freed_slots++; + } + else { + ctx->final_slots++; + } + } + break; + } + } + p += slot_size; + bitset >>= 1; + } while (bitset); +} + +static inline void +gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx) +{ + struct heap_page *sweep_page = ctx->page; + GC_ASSERT(sweep_page->heap == heap); + + uintptr_t p; + bits_t *bits, bitset; + + gc_report(2, objspace, "page_sweep: start.\n"); + +#if RGENGC_CHECK_MODE + if (!objspace->flags.immediate_sweep) { + GC_ASSERT(sweep_page->flags.before_sweep == TRUE); + } +#endif + sweep_page->flags.before_sweep = FALSE; + sweep_page->free_slots = 0; + + p = (uintptr_t)sweep_page->start; + bits = sweep_page->mark_bits; + short slot_size = sweep_page->slot_size; + int total_slots = sweep_page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + + int out_of_range_bits = total_slots % BITS_BITLENGTH; + if (out_of_range_bits != 0) { + bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1); + } + + // Clear wb_unprotected and age bits for all unmarked slots + { + bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits; + bits_t *age_bits = sweep_page->age_bits; + for (int i = 0; i < bitmap_plane_count; i++) { + bits_t unmarked = ~bits[i]; + wb_unprotected_bits[i] &= ~unmarked; + age_bits[i * 2] &= ~unmarked; + age_bits[i * 2 + 1] &= ~unmarked; + } + } + + for (int i = 0; i < bitmap_plane_count; i++) { + bitset = ~bits[i]; + if (bitset) { + gc_sweep_plane(objspace, heap, p, bitset, ctx); + } + p += BITS_BITLENGTH * slot_size; + } + + if (!heap->compact_cursor) { + gc_setup_mark_bits(sweep_page); + } + +#if GC_PROFILE_MORE_DETAIL + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->removing_objects += ctx->final_slots + ctx->freed_slots; + record->empty_objects += ctx->empty_slots; + } +#endif + if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n", + rb_gc_count(), + sweep_page->total_slots, + ctx->freed_slots, ctx->empty_slots, ctx->final_slots); + + sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots; + sweep_page->heap->total_freed_objects += ctx->freed_slots; + + if (heap_pages_deferred_final && !finalizing) { + gc_finalize_deferred_register(objspace); + } + +#if RGENGC_CHECK_MODE + short freelist_len = 0; + asan_unlock_freelist(sweep_page); + struct free_slot *ptr = sweep_page->freelist; + while (ptr) { + freelist_len++; + rb_asan_unpoison_object((VALUE)ptr, false); + struct free_slot *next = ptr->next; + rb_asan_poison_object((VALUE)ptr); + ptr = next; + } + asan_lock_freelist(sweep_page); + if (freelist_len != sweep_page->free_slots) { + rb_bug("inconsistent freelist length: expected %d but was %d", sweep_page->free_slots, freelist_len); + } +#endif + + gc_report(2, objspace, "page_sweep: end.\n"); +} + +static const char * +gc_mode_name(enum gc_mode mode) +{ + switch (mode) { + case gc_mode_none: return "none"; + case gc_mode_marking: return "marking"; + case gc_mode_sweeping: return "sweeping"; + case gc_mode_compacting: return "compacting"; + default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode); + } +} + +static void +gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode) +{ +#if RGENGC_CHECK_MODE + enum gc_mode prev_mode = gc_mode(objspace); + switch (prev_mode) { + case gc_mode_none: GC_ASSERT(mode == gc_mode_marking); break; + case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break; + case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break; + case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break; + } +#endif + if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode)); + gc_mode_set(objspace, mode); +} + +static void +heap_page_freelist_append(struct heap_page *page, struct free_slot *freelist) +{ + if (freelist) { + asan_unlock_freelist(page); + if (page->freelist) { + struct free_slot *p = page->freelist; + rb_asan_unpoison_object((VALUE)p, false); + while (p->next) { + struct free_slot *prev = p; + p = p->next; + rb_asan_poison_object((VALUE)prev); + rb_asan_unpoison_object((VALUE)p, false); + } + p->next = freelist; + rb_asan_poison_object((VALUE)p); + } + else { + page->freelist = freelist; + } + asan_lock_freelist(page); + } +} + +static void +gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap) +{ + heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node); + if (heap->sweeping_page) { + objspace->sweeping_heap_count++; + } + heap->free_pages = NULL; + heap->pooled_pages = NULL; + if (!objspace->flags.immediate_sweep) { + struct heap_page *page = NULL; + + ccan_list_for_each(&heap->pages, page, page_node) { + page->flags.before_sweep = TRUE; + } + } +} + +#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4 +__attribute__((noinline)) +#endif + +#if GC_CAN_COMPILE_COMPACTION +static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func); +static int compare_pinned_slots(const void *left, const void *right, void *d); +#endif + +static void +gc_ractor_newobj_cache_clear(void *c, void *data) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + rb_ractor_newobj_cache_t *newobj_cache = c; + + newobj_cache->incremental_mark_step_allocated_slots = 0; + + for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) { + + rb_ractor_newobj_heap_cache_t *cache = &newobj_cache->heap_caches[heap_idx]; + + rb_heap_t *heap = &heaps[heap_idx]; + RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, cache->allocated_objects_count); + cache->allocated_objects_count = 0; + + struct heap_page *page = cache->using_page; + struct free_slot *freelist = cache->freelist; + RUBY_DEBUG_LOG("ractor using_page:%p freelist:%p", (void *)page, (void *)freelist); + + heap_page_freelist_append(page, freelist); + + cache->using_page = NULL; + cache->freelist = NULL; + } +} + +static void +gc_sweep_freeobj_hooks_page(rb_objspace_t *objspace, struct heap_page *page) +{ + bits_t *bits = page->mark_bits; + uintptr_t p = (uintptr_t)page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + + int out_of_range_bits = total_slots % BITS_BITLENGTH; + bits_t last_plane_mask = (out_of_range_bits != 0) + ? ~(((bits_t)1 << out_of_range_bits) - 1) + : 0; + + for (int j = 0; j < bitmap_plane_count; j++) { + bits_t bitset = ~bits[j]; + if (j == bitmap_plane_count - 1) { + bitset &= ~last_plane_mask; + } + + uintptr_t pp = p; + while (bitset) { + if (bitset & 1) { + VALUE vp = (VALUE)pp; + asan_unpoisoning_object(vp) { + switch (BUILTIN_TYPE(vp)) { + case T_NONE: + case T_ZOMBIE: + case T_MOVED: + break; + default: + rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ); + break; + } + } + } + pp += slot_size; + bitset >>= 1; + } + p += BITS_BITLENGTH * slot_size; + } +} + +static void +gc_sweep_freeobj_hooks(rb_objspace_t *objspace) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + struct heap_page *page = NULL; + + ccan_list_for_each(&heap->pages, page, page_node) { + gc_sweep_freeobj_hooks_page(objspace, page); + } + } +} + +static void +gc_sweep_start(rb_objspace_t *objspace) +{ + gc_mode_transition(objspace, gc_mode_sweeping); + objspace->rincgc.pooled_slots = 0; + + if (RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_FREEOBJ)) { + gc_sweep_freeobj_hooks(objspace); + } + +#if GC_CAN_COMPILE_COMPACTION + if (objspace->flags.during_compacting) { + gc_sort_heap_by_compare_func( + objspace, + objspace->rcompactor.compare_func ? objspace->rcompactor.compare_func : compare_pinned_slots + ); + } +#endif + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + gc_sweep_start_heap(objspace, heap); + + /* We should call gc_sweep_finish_heap for size pools with no pages. */ + if (heap->sweeping_page == NULL) { + GC_ASSERT(heap->total_pages == 0); + GC_ASSERT(heap->total_slots == 0); + gc_sweep_finish_heap(objspace, heap); + } + } + + rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL); +} + +static void +gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap) +{ + size_t total_slots = heap->total_slots; + size_t swept_slots = heap->freed_slots + heap->empty_slots; + + size_t init_slots = gc_params.heap_init_bytes / heap->slot_size; + size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio); + + if (swept_slots < min_free_slots && + /* The heap is a growth heap if it freed more slots than had empty slots. */ + ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) { + /* If we don't have enough slots and we have pages on the tomb heap, move + * pages from the tomb heap to the eden heap. This may prevent page + * creation thrashing (frequently allocating and deallocting pages) and + * GC thrashing (running GC more frequently than required). */ + struct heap_page *resurrected_page; + while (swept_slots < min_free_slots && + (resurrected_page = heap_page_resurrect(objspace))) { + heap_add_page(objspace, heap, resurrected_page); + heap_add_freepage(heap, resurrected_page); + + swept_slots += resurrected_page->free_slots; + } + + if (swept_slots < min_free_slots) { + /* Grow this heap if we are in a major GC or if we haven't run at least + * RVALUE_OLD_AGE minor GC since the last major GC. */ + if (is_full_marking(objspace) || + objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) { + if (objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) { + heap_allocatable_bytes_expand(objspace, heap, swept_slots, heap->total_slots, heap->slot_size); + } + } + else if (swept_slots < min_free_slots * 7 / 8 && + objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) { + gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE; + heap->force_major_gc_count++; + } + } + } +} + +static void +gc_sweep_finish(rb_objspace_t *objspace) +{ + gc_report(1, objspace, "gc_sweep_finish\n"); + + gc_prof_set_heap_info(objspace); + heap_pages_free_unused_pages(objspace); + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + heap->freed_slots = 0; + heap->empty_slots = 0; + + if (!will_be_incremental_marking(objspace)) { + struct heap_page *end_page = heap->free_pages; + if (end_page) { + while (end_page->free_next) end_page = end_page->free_next; + end_page->free_next = heap->pooled_pages; + } + else { + heap->free_pages = heap->pooled_pages; + } + heap->pooled_pages = NULL; + objspace->rincgc.pooled_slots = 0; + } + } + + (void)gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters); +#if RGENGC_ESTIMATE_OLDMALLOC + if (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) { + (void)gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.oldcounters); + } +#endif + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP); + gc_mode_transition(objspace, gc_mode_none); + +#if RGENGC_CHECK_MODE >= 2 + gc_verify_internal_consistency(objspace); +#endif +} + +static int +gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap) +{ + struct heap_page *sweep_page = heap->sweeping_page; + int swept_slots = 0; + int pooled_slots = 0; + int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size; + int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size; + + if (sweep_page == NULL) return FALSE; + +#if GC_ENABLE_LAZY_SWEEP + gc_prof_sweep_timer_start(objspace); +#endif + + do { + RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page); + + struct gc_sweep_context ctx = { + .page = sweep_page, + .final_slots = 0, + .freed_slots = 0, + .empty_slots = 0, + }; + gc_sweep_page(objspace, heap, &ctx); + int free_slots = ctx.freed_slots + ctx.empty_slots; + + heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node); + + if (free_slots == sweep_page->total_slots) { + /* There are no living objects, so move this page to the global empty pages. */ + heap_unlink_page(objspace, heap, sweep_page); + + sweep_page->start = 0; + sweep_page->total_slots = 0; + sweep_page->slot_size = 0; + sweep_page->heap = NULL; + sweep_page->free_slots = 0; + + asan_unlock_freelist(sweep_page); + sweep_page->freelist = NULL; + asan_lock_freelist(sweep_page); + + asan_poison_memory_region(sweep_page->body, HEAP_PAGE_SIZE); + + objspace->empty_pages_count++; + sweep_page->free_next = objspace->empty_pages; + objspace->empty_pages = sweep_page; + } + else if (free_slots > 0) { + heap->freed_slots += ctx.freed_slots; + heap->empty_slots += ctx.empty_slots; + + if (pooled_slots < pool_budget) { + heap_add_poolpage(objspace, heap, sweep_page); + pooled_slots += free_slots; + } + else { + heap_add_freepage(heap, sweep_page); + swept_slots += free_slots; + if (swept_slots > sweep_budget) { + break; + } + } + } + else { + sweep_page->free_next = NULL; + } + } while ((sweep_page = heap->sweeping_page)); + + if (!heap->sweeping_page) { + objspace->sweeping_heap_count--; + GC_ASSERT(objspace->sweeping_heap_count >= 0); + gc_sweep_finish_heap(objspace, heap); + + if (!has_sweeping_pages(objspace)) { + gc_sweep_finish(objspace); + } + } + +#if GC_ENABLE_LAZY_SWEEP + gc_prof_sweep_timer_stop(objspace); +#endif + + return heap->free_pages != NULL; +} + +static void +gc_sweep_rest(rb_objspace_t *objspace) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + while (heap->sweeping_page) { + gc_sweep_step(objspace, heap); + } + } +} + +static void +gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap) +{ + GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD); + if (!GC_ENABLE_LAZY_SWEEP) return; + + gc_sweeping_enter(objspace); + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + if (gc_sweep_step(objspace, heap)) { + GC_ASSERT(heap->free_pages != NULL); + } + else if (heap == sweep_heap) { + if (objspace->empty_pages_count > 0 || objspace->heap_pages.allocatable_bytes > 0) { + /* [Bug #21548] + * + * If this heap is the heap we want to sweep, but we weren't able + * to free any slots, but we also either have empty pages or could + * allocate new pages, then we want to preemptively claim a page + * because it's possible that sweeping another heap will call + * gc_sweep_finish_heap, which may use up all of the + * empty/allocatable pages. If other heaps are not finished sweeping + * then we do not finish this GC and we will end up triggering a new + * GC cycle during this GC phase. */ + heap_page_allocate_and_initialize(objspace, heap); + + GC_ASSERT(heap->free_pages != NULL); + } + else { + /* Not allowed to create a new page so finish sweeping. */ + gc_sweep_rest(objspace); + GC_ASSERT(gc_mode(objspace) == gc_mode_none); + break; + } + } + } + + gc_sweeping_exit(objspace); +} + +VALUE +rb_gc_impl_location(void *objspace_ptr, VALUE value) +{ + VALUE destination; + + asan_unpoisoning_object(value) { + if (BUILTIN_TYPE(value) == T_MOVED) { + destination = (VALUE)RMOVED(value)->destination; + GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE); + } + else { + destination = value; + } + } + + return destination; +} + +#if GC_CAN_COMPILE_COMPACTION +static void +invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset) +{ + if (bitset) { + do { + if (bitset & 1) { + VALUE forwarding_object = (VALUE)p; + VALUE object; + + if (BUILTIN_TYPE(forwarding_object) == T_MOVED) { + GC_ASSERT(RVALUE_PINNED(objspace, forwarding_object)); + GC_ASSERT(!RVALUE_MARKED(objspace, forwarding_object)); + + CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object); + + object = rb_gc_impl_location(objspace, forwarding_object); + gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object), page); + /* forwarding_object is now our actual object, and "object" + * is the free slot for the original page */ + + struct heap_page *orig_page = GET_HEAP_PAGE(object); + orig_page->free_slots++; + RVALUE_AGE_SET_BITMAP(object, 0); + heap_page_add_freeobj(objspace, orig_page, object); + + GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object)); + GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED); + GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE); + } + } + p += page->slot_size; + bitset >>= 1; + } while (bitset); + } +} + +static void +invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page) +{ + int i; + bits_t *mark_bits, *pin_bits; + bits_t bitset; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + + mark_bits = page->mark_bits; + pin_bits = page->pinned_bits; + + uintptr_t p = page->start; + + for (i=0; i < bitmap_plane_count; i++) { + /* Moved objects are pinned but never marked. We reuse the pin bits + * to indicate there is a moved object in this slot. */ + bitset = pin_bits[i] & ~mark_bits[i]; + invalidate_moved_plane(objspace, page, p, bitset); + p += BITS_BITLENGTH * slot_size; + } +} +#endif + +static void +gc_compact_start(rb_objspace_t *objspace) +{ + struct heap_page *page = NULL; + gc_mode_transition(objspace, gc_mode_compacting); + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + ccan_list_for_each(&heap->pages, page, page_node) { + page->flags.before_sweep = TRUE; + } + + heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node); + heap->compact_cursor_index = 0; + } + + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->moved_objects = objspace->rcompactor.total_moved; + } + + memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t)); + memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t)); + memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t)); + memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t)); + + /* Set up read barrier for pages containing MOVED objects */ + install_handlers(); +} + +static void gc_sweep_compact(rb_objspace_t *objspace); + +static void +gc_sweep(rb_objspace_t *objspace) +{ + gc_sweeping_enter(objspace); + + const unsigned int immediate_sweep = objspace->flags.immediate_sweep; + + gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep); + + gc_sweep_start(objspace); + if (objspace->flags.during_compacting) { + gc_sweep_compact(objspace); + } + + if (immediate_sweep) { +#if !GC_ENABLE_LAZY_SWEEP + gc_prof_sweep_timer_start(objspace); +#endif + gc_sweep_rest(objspace); +#if !GC_ENABLE_LAZY_SWEEP + gc_prof_sweep_timer_stop(objspace); +#endif + } + else { + + /* Sweep every size pool. */ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + gc_sweep_step(objspace, heap); + } + } + + gc_sweeping_exit(objspace); +} + +/* Marking - Marking stack */ + +static stack_chunk_t * +stack_chunk_alloc(void) +{ + stack_chunk_t *res; + + res = malloc(sizeof(stack_chunk_t)); + if (!res) + rb_memerror(); + + return res; +} + +static inline int +is_mark_stack_empty(mark_stack_t *stack) +{ + return stack->chunk == NULL; +} + +static size_t +mark_stack_size(mark_stack_t *stack) +{ + size_t size = stack->index; + stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL; + + while (chunk) { + size += stack->limit; + chunk = chunk->next; + } + return size; +} + +static void +add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk) +{ + chunk->next = stack->cache; + stack->cache = chunk; + stack->cache_size++; +} + +static void +shrink_stack_chunk_cache(mark_stack_t *stack) +{ + stack_chunk_t *chunk; + + if (stack->unused_cache_size > (stack->cache_size/2)) { + chunk = stack->cache; + stack->cache = stack->cache->next; + stack->cache_size--; + free(chunk); + } + stack->unused_cache_size = stack->cache_size; +} + +static void +push_mark_stack_chunk(mark_stack_t *stack) +{ + stack_chunk_t *next; + + GC_ASSERT(stack->index == stack->limit); + + if (stack->cache_size > 0) { + next = stack->cache; + stack->cache = stack->cache->next; + stack->cache_size--; + if (stack->unused_cache_size > stack->cache_size) + stack->unused_cache_size = stack->cache_size; + } + else { + next = stack_chunk_alloc(); + } + next->next = stack->chunk; + stack->chunk = next; + stack->index = 0; +} + +static void +pop_mark_stack_chunk(mark_stack_t *stack) +{ + stack_chunk_t *prev; + + prev = stack->chunk->next; + GC_ASSERT(stack->index == 0); + add_stack_chunk_cache(stack, stack->chunk); + stack->chunk = prev; + stack->index = stack->limit; +} + +static void +mark_stack_chunk_list_free(stack_chunk_t *chunk) +{ + stack_chunk_t *next = NULL; + + while (chunk != NULL) { + next = chunk->next; + free(chunk); + chunk = next; + } +} + +static void +free_stack_chunks(mark_stack_t *stack) +{ + mark_stack_chunk_list_free(stack->chunk); +} + +static void +mark_stack_free_cache(mark_stack_t *stack) +{ + mark_stack_chunk_list_free(stack->cache); + stack->cache_size = 0; + stack->unused_cache_size = 0; +} + +static void +push_mark_stack(mark_stack_t *stack, VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + case T_CLASS: + case T_MODULE: + case T_FLOAT: + case T_STRING: + case T_REGEXP: + case T_ARRAY: + case T_HASH: + case T_STRUCT: + case T_BIGNUM: + case T_FILE: + case T_DATA: + case T_MATCH: + case T_COMPLEX: + case T_RATIONAL: + case T_TRUE: + case T_FALSE: + case T_SYMBOL: + case T_IMEMO: + case T_ICLASS: + if (stack->index == stack->limit) { + push_mark_stack_chunk(stack); + } + stack->chunk->data[stack->index++] = obj; + return; + + case T_NONE: + case T_NIL: + case T_FIXNUM: + case T_MOVED: + case T_ZOMBIE: + case T_UNDEF: + case T_MASK: + rb_bug("push_mark_stack() called for broken object"); + break; + + case T_NODE: + rb_bug("push_mark_stack: unexpected T_NODE object"); + break; + } + + rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s", + BUILTIN_TYPE(obj), (void *)obj, + is_pointer_to_heap((rb_objspace_t *)rb_gc_get_objspace(), (void *)obj) ? "corrupted object" : "non object"); +} + +static int +pop_mark_stack(mark_stack_t *stack, VALUE *data) +{ + if (is_mark_stack_empty(stack)) { + return FALSE; + } + if (stack->index == 1) { + *data = stack->chunk->data[--stack->index]; + pop_mark_stack_chunk(stack); + } + else { + *data = stack->chunk->data[--stack->index]; + } + return TRUE; +} + +static void +init_mark_stack(mark_stack_t *stack) +{ + int i; + + MEMZERO(stack, mark_stack_t, 1); + stack->index = stack->limit = STACK_CHUNK_SIZE; + + for (i=0; i < 4; i++) { + add_stack_chunk_cache(stack, stack_chunk_alloc()); + } + stack->unused_cache_size = stack->cache_size; +} + +/* Marking */ + +static void +rgengc_check_relation(rb_objspace_t *objspace, VALUE obj) +{ + if (objspace->rgengc.parent_object_old_p) { + if (RVALUE_WB_UNPROTECTED(objspace, obj) || !RVALUE_OLD_P(objspace, obj)) { + rgengc_remember(objspace, objspace->rgengc.parent_object); + } + } +} + +static inline int +gc_mark_set(rb_objspace_t *objspace, VALUE obj) +{ + if (RVALUE_MARKED(objspace, obj)) return 0; + MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj); + return 1; +} + +static void +gc_aging(rb_objspace_t *objspace, VALUE obj) +{ + /* Disable aging if Major GC's are disabled. This will prevent longish lived + * objects filling up the heap at the expense of marking many more objects. + * + * We should always pre-warm our process when disabling majors, by running + * GC manually several times so that most objects likely to become oldgen + * are already oldgen. + */ + if(!gc_config_full_mark_val) + return; + + struct heap_page *page = GET_HEAP_PAGE(obj); + + GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE); + check_rvalue_consistency(objspace, obj); + + if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) { + if (!RVALUE_OLD_P(objspace, obj)) { + int t = BUILTIN_TYPE(obj); + if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) { + gc_report(3, objspace, "gc_aging: YOUNG class: %s\n", rb_obj_info(obj)); + RVALUE_AGE_SET(obj, RVALUE_OLD_AGE); + RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj); + } + else { + gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj)); + RVALUE_AGE_INC(objspace, obj); + } + } + else if (is_full_marking(objspace)) { + GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE); + RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj); + } + } + check_rvalue_consistency(objspace, obj); + + objspace->marked_slots++; +} + +static void +gc_grey(rb_objspace_t *objspace, VALUE obj) +{ +#if RGENGC_CHECK_MODE + if (RVALUE_MARKED(objspace, obj) == FALSE) rb_bug("gc_grey: %s is not marked.", rb_obj_info(obj)); + if (RVALUE_MARKING(objspace, obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", rb_obj_info(obj)); +#endif + + if (is_incremental_marking(objspace)) { + MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj); + } + + if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) { + rb_darray_append_without_gc(&objspace->weak_references, obj); + } + + push_mark_stack(&objspace->mark_stack, obj); +} + +static inline void +gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj) +{ + if (RB_UNLIKELY(BUILTIN_TYPE(obj) == T_NONE)) { + enum {info_size = 256}; + char obj_info_buf[info_size]; + rb_raw_obj_info(obj_info_buf, info_size, obj); + + char parent_obj_info_buf[info_size]; + rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object); + + rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf); + } +} + +static void +gc_mark(rb_objspace_t *objspace, VALUE obj) +{ + GC_ASSERT(during_gc); + GC_ASSERT(!objspace->flags.during_reference_updating); + + rgengc_check_relation(objspace, obj); + if (!gc_mark_set(objspace, obj)) return; /* already marked */ + + if (0) { // for debug GC marking miss + RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)", + (void *)obj, obj_type_name(obj), + (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object)); + } + + gc_mark_check_t_none(objspace, obj); + + gc_aging(objspace, obj); + gc_grey(objspace, obj); +} + +static inline void +gc_pin(rb_objspace_t *objspace, VALUE obj) +{ + GC_ASSERT(!SPECIAL_CONST_P(obj)); + if (RB_UNLIKELY(objspace->flags.during_compacting)) { + if (RB_LIKELY(during_gc)) { + if (!RVALUE_PINNED(objspace, obj)) { + GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots); + GET_HEAP_PAGE(obj)->pinned_slots++; + MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj); + } + } + } +} + +static inline void +gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj) +{ + gc_pin(objspace, obj); + gc_mark(objspace, obj); +} + +void +rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (RB_UNLIKELY(objspace->flags.during_reference_updating)) { + GC_ASSERT(objspace->flags.during_compacting); + GC_ASSERT(during_gc); + + VALUE destination = rb_gc_impl_location(objspace, *ptr); + if (destination != *ptr) { + *ptr = destination; + } + } + else { + gc_mark(objspace, *ptr); + } +} + +void +rb_gc_impl_mark(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + gc_mark(objspace, obj); +} + +void +rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + gc_mark_and_pin(objspace, obj); +} + +void +rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj)); + + if (is_pointer_to_heap(objspace, (void *)obj)) { + asan_unpoisoning_object(obj) { + /* Garbage can live on the stack, so do not mark or pin */ + switch (BUILTIN_TYPE(obj)) { + case T_ZOMBIE: + case T_NONE: + break; + default: + gc_mark_and_pin(objspace, obj); + break; + } + } + } +} + +static int +pin_value(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); + + return ST_CONTINUE; +} + +static inline void +gc_mark_set_parent_raw(rb_objspace_t *objspace, VALUE obj, bool old_p) +{ + asan_unpoison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object), false); + asan_unpoison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p), false); + objspace->rgengc.parent_object = obj; + objspace->rgengc.parent_object_old_p = old_p; +} + +static inline void +gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj) +{ + gc_mark_set_parent_raw(objspace, obj, RVALUE_OLD_P(objspace, obj)); +} + +static inline void +gc_mark_set_parent_invalid(rb_objspace_t *objspace) +{ + asan_poison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object)); + asan_poison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p)); +} + +static void +mark_roots(rb_objspace_t *objspace, const char **categoryp) +{ +#define MARK_CHECKPOINT(category) do { \ + if (categoryp) *categoryp = category; \ +} while (0) + + MARK_CHECKPOINT("objspace"); + gc_mark_set_parent_raw(objspace, Qundef, false); + + if (finalizer_table != NULL) { + st_foreach(finalizer_table, pin_value, (st_data_t)objspace); + } + + if (stress_to_class) rb_gc_mark(stress_to_class); + + rb_gc_save_machine_context(); + rb_gc_mark_roots(objspace, categoryp); + gc_mark_set_parent_invalid(objspace); +} + +static void +gc_mark_children(rb_objspace_t *objspace, VALUE obj) +{ + gc_mark_set_parent(objspace, obj); + rb_gc_mark_children(objspace, obj); + gc_mark_set_parent_invalid(objspace); +} + +/** + * incremental: 0 -> not incremental (do all) + * incremental: n -> mark at most `n' objects + */ +static inline int +gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count) +{ + mark_stack_t *mstack = &objspace->mark_stack; + VALUE obj; + size_t marked_slots_at_the_beginning = objspace->marked_slots; + size_t popped_count = 0; + + while (pop_mark_stack(mstack, &obj)) { + if (obj == Qundef) continue; /* skip */ + + if (RGENGC_CHECK_MODE && !RVALUE_MARKED(objspace, obj)) { + rb_bug("gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj)); + } + gc_mark_children(objspace, obj); + + if (incremental) { + if (RGENGC_CHECK_MODE && !RVALUE_MARKING(objspace, obj)) { + rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0"); + } + CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj); + popped_count++; + + if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) { + break; + } + } + else { + /* just ignore marking bits */ + } + } + + if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace); + + if (is_mark_stack_empty(mstack)) { + shrink_stack_chunk_cache(mstack); + return TRUE; + } + else { + return FALSE; + } +} + +static int +gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count) +{ + return gc_mark_stacked_objects(objspace, TRUE, count); +} + +static int +gc_mark_stacked_objects_all(rb_objspace_t *objspace) +{ + return gc_mark_stacked_objects(objspace, FALSE, 0); +} + +#if RGENGC_CHECK_MODE >= 4 + +#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01) +#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01) +#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1)) + +struct reflist { + VALUE *list; + int pos; + int size; +}; + +static struct reflist * +reflist_create(VALUE obj) +{ + struct reflist *refs = xmalloc(sizeof(struct reflist)); + refs->size = 1; + refs->list = ALLOC_N(VALUE, refs->size); + refs->list[0] = obj; + refs->pos = 1; + return refs; +} + +static void +reflist_destruct(struct reflist *refs) +{ + xfree(refs->list); + xfree(refs); +} + +static void +reflist_add(struct reflist *refs, VALUE obj) +{ + if (refs->pos == refs->size) { + refs->size *= 2; + SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2); + } + + refs->list[refs->pos++] = obj; +} + +static void +reflist_dump(struct reflist *refs) +{ + int i; + for (i=0; i<refs->pos; i++) { + VALUE obj = refs->list[i]; + if (IS_ROOTSIG(obj)) { /* root */ + fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj)); + } + else { + fprintf(stderr, "<%s>", rb_obj_info(obj)); + } + if (i+1 < refs->pos) fprintf(stderr, ", "); + } +} + +static int +reflist_referred_from_machine_context(struct reflist *refs) +{ + int i; + for (i=0; i<refs->pos; i++) { + VALUE obj = refs->list[i]; + if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1; + } + return 0; +} + +struct allrefs { + rb_objspace_t *objspace; + /* a -> obj1 + * b -> obj1 + * c -> obj1 + * c -> obj2 + * d -> obj3 + * #=> {obj1 => [a, b, c], obj2 => [c, d]} + */ + struct st_table *references; + const char *category; + VALUE root_obj; + mark_stack_t mark_stack; +}; + +static int +allrefs_add(struct allrefs *data, VALUE obj) +{ + struct reflist *refs; + st_data_t r; + + if (st_lookup(data->references, obj, &r)) { + refs = (struct reflist *)r; + reflist_add(refs, data->root_obj); + return 0; + } + else { + refs = reflist_create(data->root_obj); + st_insert(data->references, obj, (st_data_t)refs); + return 1; + } +} + +static void +allrefs_i(VALUE obj, void *ptr) +{ + struct allrefs *data = (struct allrefs *)ptr; + + if (allrefs_add(data, obj)) { + push_mark_stack(&data->mark_stack, obj); + } +} + +static void +allrefs_roots_i(VALUE obj, void *ptr) +{ + struct allrefs *data = (struct allrefs *)ptr; + if (strlen(data->category) == 0) rb_bug("!!!"); + data->root_obj = MAKE_ROOTSIG(data->category); + + if (allrefs_add(data, obj)) { + push_mark_stack(&data->mark_stack, obj); + } +} +#define PUSH_MARK_FUNC_DATA(v) do { \ + struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \ + GET_VM()->gc.mark_func_data = (v); + +#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0) + +static st_table * +objspace_allrefs(rb_objspace_t *objspace) +{ + struct allrefs data; + struct gc_mark_func_data_struct mfd; + VALUE obj; + int prev_dont_gc = dont_gc_val(); + dont_gc_on(); + + data.objspace = objspace; + data.references = st_init_numtable(); + init_mark_stack(&data.mark_stack); + + mfd.mark_func = allrefs_roots_i; + mfd.data = &data; + + /* traverse root objects */ + PUSH_MARK_FUNC_DATA(&mfd); + GET_VM()->gc.mark_func_data = &mfd; + mark_roots(objspace, &data.category); + POP_MARK_FUNC_DATA(); + + /* traverse rest objects reachable from root objects */ + while (pop_mark_stack(&data.mark_stack, &obj)) { + rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data); + } + free_stack_chunks(&data.mark_stack); + + dont_gc_set(prev_dont_gc); + return data.references; +} + +static int +objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr) +{ + struct reflist *refs = (struct reflist *)value; + reflist_destruct(refs); + return ST_CONTINUE; +} + +static void +objspace_allrefs_destruct(struct st_table *refs) +{ + st_foreach(refs, objspace_allrefs_destruct_i, 0); + st_free_table(refs); +} + +#if RGENGC_CHECK_MODE >= 5 +static int +allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr) +{ + VALUE obj = (VALUE)k; + struct reflist *refs = (struct reflist *)v; + fprintf(stderr, "[allrefs_dump_i] %s <- ", rb_obj_info(obj)); + reflist_dump(refs); + fprintf(stderr, "\n"); + return ST_CONTINUE; +} + +static void +allrefs_dump(rb_objspace_t *objspace) +{ + VALUE size = objspace->rgengc.allrefs_table->num_entries; + fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size); + st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0); +} +#endif + +static int +gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr) +{ + VALUE obj = k; + struct reflist *refs = (struct reflist *)v; + rb_objspace_t *objspace = (rb_objspace_t *)ptr; + + /* object should be marked or oldgen */ + if (!RVALUE_MARKED(objspace, obj)) { + fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj)); + fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj); + reflist_dump(refs); + + if (reflist_referred_from_machine_context(refs)) { + fprintf(stderr, " (marked from machine stack).\n"); + /* marked from machine context can be false positive */ + } + else { + objspace->rgengc.error_count++; + fprintf(stderr, "\n"); + } + } + return ST_CONTINUE; +} + +static void +gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name) +{ + MALLOC_COUNTERS_LOCK(objspace); + struct gc_malloc_bytes saved_malloc = { + .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc), + .free = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free), + .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc), + .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc), + }; +#if RGENGC_ESTIMATE_OLDMALLOC + struct gc_malloc_bytes saved_oldmalloc = { + .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.malloc), + .free = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.free), + .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.malloc_at_last_gc), + .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.free_at_last_gc), + }; +#endif + MALLOC_COUNTERS_UNLOCK(objspace); + VALUE already_disabled = rb_objspace_gc_disable(objspace); + + objspace->rgengc.allrefs_table = objspace_allrefs(objspace); + + if (checker_func) { + st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace); + } + + if (objspace->rgengc.error_count > 0) { +#if RGENGC_CHECK_MODE >= 5 + allrefs_dump(objspace); +#endif + if (checker_name) rb_bug("%s: GC has problem.", checker_name); + } + + objspace_allrefs_destruct(objspace->rgengc.allrefs_table); + objspace->rgengc.allrefs_table = 0; + + if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace); + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_store_release(&objspace->malloc_counters.counters.malloc, saved_malloc.malloc); + gc_counter_store_release(&objspace->malloc_counters.counters.free, saved_malloc.free); + gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc); + gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc); +#if RGENGC_ESTIMATE_OLDMALLOC + gc_counter_store_release(&objspace->malloc_counters.oldcounters.malloc, saved_oldmalloc.malloc); + gc_counter_store_release(&objspace->malloc_counters.oldcounters.free, saved_oldmalloc.free); + gc_counter_store_release(&objspace->malloc_counters.oldcounters.malloc_at_last_gc, saved_oldmalloc.malloc_at_last_gc); + gc_counter_store_release(&objspace->malloc_counters.oldcounters.free_at_last_gc, saved_oldmalloc.free_at_last_gc); +#endif + MALLOC_COUNTERS_UNLOCK(objspace); +} +#endif /* RGENGC_CHECK_MODE >= 4 */ + +struct verify_internal_consistency_struct { + rb_objspace_t *objspace; + int err_count; + size_t live_object_count; + size_t zombie_object_count; + + VALUE parent; + size_t old_object_count; + size_t remembered_shady_count; +}; + +static void +check_generation_i(const VALUE child, void *ptr) +{ + struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr; + const VALUE parent = data->parent; + + if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent)); + + if (!RVALUE_OLD_P(data->objspace, child)) { + if (!RVALUE_REMEMBERED(data->objspace, parent) && + !RVALUE_REMEMBERED(data->objspace, child) && + !RVALUE_UNCOLLECTIBLE(data->objspace, child)) { + fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child)); + data->err_count++; + } + } +} + +static void +check_color_i(const VALUE child, void *ptr) +{ + struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr; + const VALUE parent = data->parent; + + if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) { + fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n", + rb_obj_info(parent), rb_obj_info(child)); + data->err_count++; + } +} + +static void +check_children_i(const VALUE child, void *ptr) +{ + struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr; + if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) { + fprintf(stderr, "check_children_i: %s has error (referenced from %s)", + rb_obj_info(child), rb_obj_info(data->parent)); + + data->err_count++; + } +} + +static int +verify_internal_consistency_i(void *page_start, void *page_end, size_t stride, + struct verify_internal_consistency_struct *data) +{ + VALUE obj; + rb_objspace_t *objspace = data->objspace; + + for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) { + asan_unpoisoning_object(obj) { + if (!rb_gc_impl_garbage_object_p(objspace, obj)) { + /* count objects */ + data->live_object_count++; + data->parent = obj; + + /* Normally, we don't expect T_MOVED objects to be in the heap. + * But they can stay alive on the stack, */ + if (!gc_object_moved_p(objspace, obj)) { + /* moved slots don't have children */ + rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data); + } + + /* check health of children */ + if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++; + if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++; + + if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) { + /* reachable objects from an oldgen object should be old or (young with remember) */ + data->parent = obj; + rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data); + } + + if (!is_marking(objspace) && rb_gc_obj_shareable_p(obj)) { + rb_gc_verify_shareable(obj); + } + + if (is_incremental_marking(objspace)) { + if (RVALUE_BLACK_P(objspace, obj)) { + /* reachable objects from black objects should be black or grey objects */ + data->parent = obj; + rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data); + } + } + } + else { + if (BUILTIN_TYPE(obj) == T_ZOMBIE) { + data->zombie_object_count++; + + if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) { + fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n", + rb_obj_info(obj)); + data->err_count++; + } + + if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) { + fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n", + FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in", + rb_obj_info(obj)); + data->err_count++; + } + } + } + } + } + + return 0; +} + +static int +gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) +{ + unsigned int has_remembered_shady = FALSE; + unsigned int has_remembered_old = FALSE; + int remembered_old_objects = 0; + int free_objects = 0; + int zombie_objects = 0; + + short slot_size = page->slot_size; + uintptr_t start = (uintptr_t)page->start; + uintptr_t end = start + page->total_slots * slot_size; + + for (uintptr_t ptr = start; ptr < end; ptr += slot_size) { + VALUE val = (VALUE)ptr; + asan_unpoisoning_object(val) { + enum ruby_value_type type = BUILTIN_TYPE(val); + + if (type == T_NONE) free_objects++; + if (type == T_ZOMBIE) zombie_objects++; + if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) { + has_remembered_shady = TRUE; + } + if (RVALUE_PAGE_MARKING(page, val)) { + has_remembered_old = TRUE; + remembered_old_objects++; + } + } + } + + if (!is_incremental_marking(objspace) && + page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) { + + for (uintptr_t ptr = start; ptr < end; ptr += slot_size) { + VALUE val = (VALUE)ptr; + if (RVALUE_PAGE_MARKING(page, val)) { + fprintf(stderr, "marking -> %s\n", rb_obj_info(val)); + } + } + rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s", + (void *)page, remembered_old_objects, obj ? rb_obj_info(obj) : ""); + } + + if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) { + rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s", + (void *)page, obj ? rb_obj_info(obj) : ""); + } + + if (0) { + /* free_slots may not equal to free_objects */ + if (page->free_slots != free_objects) { + rb_bug("page %p's free_slots should be %d, but %d", (void *)page, page->free_slots, free_objects); + } + } + if (page->final_slots != zombie_objects) { + rb_bug("page %p's final_slots should be %d, but %d", (void *)page, page->final_slots, zombie_objects); + } + + return remembered_old_objects; +} + +static int +gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head) +{ + int remembered_old_objects = 0; + struct heap_page *page = 0; + + ccan_list_for_each(head, page, page_node) { + asan_unlock_freelist(page); + struct free_slot *p = page->freelist; + while (p) { + VALUE vp = (VALUE)p; + VALUE prev = vp; + rb_asan_unpoison_object(vp, false); + if (BUILTIN_TYPE(vp) != T_NONE) { + fprintf(stderr, "freelist slot expected to be T_NONE but was: %s\n", rb_obj_info(vp)); + } + p = p->next; + rb_asan_poison_object(prev); + } + asan_lock_freelist(page); + + if (page->flags.has_remembered_objects == FALSE) { + remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse); + } + } + + return remembered_old_objects; +} + +static int +gc_verify_heap_pages(rb_objspace_t *objspace) +{ + int remembered_old_objects = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + remembered_old_objects += gc_verify_heap_pages_(objspace, &((&heaps[i])->pages)); + } + return remembered_old_objects; +} + +static void +gc_verify_internal_consistency_(rb_objspace_t *objspace) +{ + struct verify_internal_consistency_struct data = {0}; + + data.objspace = objspace; + gc_report(5, objspace, "gc_verify_internal_consistency: start\n"); + + /* check relations */ + for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { + struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); + short slot_size = page->slot_size; + + uintptr_t start = (uintptr_t)page->start; + uintptr_t end = start + page->total_slots * slot_size; + + verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data); + } + + if (data.err_count != 0) { +#if RGENGC_CHECK_MODE >= 5 + objspace->rgengc.error_count = data.err_count; + gc_marks_check(objspace, NULL, NULL); + allrefs_dump(objspace); +#endif + rb_bug("gc_verify_internal_consistency: found internal inconsistency."); + } + + /* check heap_page status */ + gc_verify_heap_pages(objspace); + + /* check counters */ + + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + + if (!is_lazy_sweeping(objspace) && + !finalizing && + !rb_gc_multi_ractor_p()) { + if (objspace_live_slots(objspace) != data.live_object_count) { + fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", total_freed_objects: %"PRIdSIZE"\n", + total_final_slots_count(objspace), total_freed_objects(objspace)); + rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".", + objspace_live_slots(objspace), data.live_object_count); + } + } + + if (!is_marking(objspace)) { + if (objspace->rgengc.old_objects != data.old_object_count) { + rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".", + objspace->rgengc.old_objects, data.old_object_count); + } + if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) { + rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".", + objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count); + } + } + + if (!finalizing) { + size_t list_count = 0; + + { + VALUE z = heap_pages_deferred_final; + while (z) { + list_count++; + z = RZOMBIE(z)->next; + } + } + + if (total_final_slots_count(objspace) != data.zombie_object_count || + total_final_slots_count(objspace) != list_count) { + + rb_bug("inconsistent finalizing object count:\n" + " expect %"PRIuSIZE"\n" + " but %"PRIuSIZE" zombies\n" + " heap_pages_deferred_final list has %"PRIuSIZE" items.", + total_final_slots_count(objspace), + data.zombie_object_count, + list_count); + } + } + + gc_report(5, objspace, "gc_verify_internal_consistency: OK\n"); +} + +static void +gc_verify_internal_consistency(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + unsigned int lev = RB_GC_VM_LOCK(); + { + rb_gc_vm_barrier(); // stop other ractors + + unsigned int prev_during_gc = during_gc; + during_gc = FALSE; // stop gc here + { + gc_verify_internal_consistency_(objspace); + } + during_gc = prev_during_gc; + } + RB_GC_VM_UNLOCK(lev); +} + +static void +heap_move_pooled_pages_to_free_pages(rb_heap_t *heap) +{ + if (heap->pooled_pages) { + if (heap->free_pages) { + struct heap_page *free_pages_tail = heap->free_pages; + while (free_pages_tail->free_next) { + free_pages_tail = free_pages_tail->free_next; + } + free_pages_tail->free_next = heap->pooled_pages; + } + else { + heap->free_pages = heap->pooled_pages; + } + + heap->pooled_pages = NULL; + } +} + +static int +gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj) +{ + struct heap_page *page = GET_HEAP_PAGE(obj); + bits_t *uncollectible_bits = &page->uncollectible_bits[0]; + + if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) { + page->flags.has_uncollectible_wb_unprotected_objects = TRUE; + MARK_IN_BITMAP(uncollectible_bits, obj); + objspace->rgengc.uncollectible_wb_unprotected_objects++; + +#if RGENGC_PROFILE > 0 + objspace->profile.total_remembered_shady_object_count++; +#if RGENGC_PROFILE >= 2 + objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++; +#endif +#endif + return TRUE; + } + else { + return FALSE; + } +} + +static inline void +gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits, short slot_size) +{ + if (bits) { + do { + if (bits & 1) { + gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((VALUE)p)); + GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, (VALUE)p)); + GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p)); + gc_mark_children(objspace, (VALUE)p); + } + p += slot_size; + bits >>= 1; + } while (bits); + } +} + +static void +gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap) +{ + struct heap_page *page = 0; + + ccan_list_for_each(&heap->pages, page, page_node) { + bits_t *mark_bits = page->mark_bits; + bits_t *wbun_bits = page->wb_unprotected_bits; + uintptr_t p = page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + size_t j; + + for (j=0; j<(size_t)bitmap_plane_count; j++) { + bits_t bits = mark_bits[j] & wbun_bits[j]; + gc_marks_wb_unprotected_objects_plane(objspace, p, bits, slot_size); + p += BITS_BITLENGTH * slot_size; + } + } + + gc_mark_stacked_objects_all(objspace); +} + +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) +{ + FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE); +} + +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + bool marked = RVALUE_MARKED(objspace, obj); + + if (marked) { + rgengc_check_relation(objspace, obj); + } + + return marked; +} + +static void +gc_update_weak_references(rb_objspace_t *objspace) +{ + VALUE *obj_ptr; + rb_darray_foreach(objspace->weak_references, i, obj_ptr) { + gc_mark_set_parent(objspace, *obj_ptr); + rb_gc_handle_weak_references(*obj_ptr); + gc_mark_set_parent_invalid(objspace); + } + + size_t capa = rb_darray_capa(objspace->weak_references); + size_t size = rb_darray_size(objspace->weak_references); + + objspace->profile.weak_references_count = size; + + rb_darray_clear(objspace->weak_references); + + /* If the darray has capacity for more than four times the amount used, we + * shrink it down to half of that capacity. */ + if (capa > size * 4) { + rb_darray_resize_capa_without_gc(&objspace->weak_references, size * 2); + } +} + +static void +gc_marks_finish(rb_objspace_t *objspace) +{ + /* finish incremental GC */ + if (is_incremental_marking(objspace)) { + if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) { + rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").", + mark_stack_size(&objspace->mark_stack)); + } + + mark_roots(objspace, NULL); + while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false); + +#if RGENGC_CHECK_MODE >= 2 + if (gc_verify_heap_pages(objspace) != 0) { + rb_bug("gc_marks_finish (incremental): there are remembered old objects."); + } +#endif + + objspace->flags.during_incremental_marking = FALSE; + /* check children of all marked wb-unprotected objects */ + for (int i = 0; i < HEAP_COUNT; i++) { + gc_marks_wb_unprotected_objects(objspace, &heaps[i]); + } + } + + gc_update_weak_references(objspace); + +#if RGENGC_CHECK_MODE >= 2 + gc_verify_internal_consistency(objspace); +#endif + +#if RGENGC_CHECK_MODE >= 4 + during_gc = FALSE; + gc_marks_check(objspace, gc_check_after_marks_i, "after_marks"); + during_gc = TRUE; +#endif + + { + const unsigned long r_mul = objspace->live_ractor_cache_count > 8 ? 8 : objspace->live_ractor_cache_count; // upto 8 + + size_t total_slots = objspace_available_slots(objspace); + size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */ + size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio); + size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio); + if (min_free_slots < gc_params.heap_free_slots * r_mul) { + min_free_slots = gc_params.heap_free_slots * r_mul; + } + + int full_marking = is_full_marking(objspace); + + GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots); + + /* Setup freeable slots. */ + size_t total_init_slots = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + total_init_slots += (gc_params.heap_init_bytes / heaps[i].slot_size) * r_mul; + } + + if (max_free_slots < total_init_slots) { + max_free_slots = total_init_slots; + } + + /* Approximate freeable pages using the average slots-per-pages across all heaps */ + if (sweep_slots > max_free_slots) { + size_t excess_slots = sweep_slots - max_free_slots; + size_t total_heap_pages = heap_eden_total_pages(objspace); + heap_pages_freeable_pages = total_heap_pages > 0 + ? excess_slots * total_heap_pages / total_slots + : 0; + } + else { + heap_pages_freeable_pages = 0; + } + + if (objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) { + if (!full_marking && sweep_slots < min_free_slots * 7 / 8) { + if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) { + full_marking = TRUE; + } + else { + gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n"); + gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE; + } + } + + if (full_marking) { + heap_allocatable_bytes_expand(objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size); + } + } + + if (full_marking) { + /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */ + const double r = gc_params.oldobject_limit_factor; + objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX( + (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r), + (size_t)(objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio) + ); + objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r); + } + + if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) { + gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY; + } + if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) { + gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN; + } + + gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, " + "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, " + "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" bytes, next GC: %s)\n", + objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_bytes, + gc_needs_major_flags ? "major" : "minor"); + } + + // TODO: refactor so we don't need to call this + rb_ractor_finish_marking(); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_MARK); +} + +static bool +gc_compact_heap_cursors_met_p(rb_heap_t *heap) +{ + return heap->sweeping_page == heap->compact_cursor; +} + + +static rb_heap_t * +gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE obj) +{ + size_t obj_size = rb_gc_obj_optimal_size(obj); + if (obj_size == 0) { + return src_pool; + } + + GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size)); + + size_t idx = heap_idx_for_size(obj_size); + + return &heaps[idx]; +} + +static bool +gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src) +{ + GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED); + GC_ASSERT(gc_is_moveable_obj(objspace, src)); + + rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src); + if (gc_compact_heap_cursors_met_p(dest_pool)) { + return dest_pool != heap; + } + + while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) { + struct gc_sweep_context ctx = { + .page = dest_pool->sweeping_page, + .final_slots = 0, + .freed_slots = 0, + .empty_slots = 0, + }; + + /* The page of src could be partially compacted, so it may contain + * T_MOVED. Sweeping a page may read objects on this page, so we + * need to lock the page. */ + lock_page_body(objspace, GET_PAGE_BODY(src)); + gc_sweep_page(objspace, dest_pool, &ctx); + unlock_page_body(objspace, GET_PAGE_BODY(src)); + + if (dest_pool->sweeping_page->free_slots > 0) { + heap_add_freepage(dest_pool, dest_pool->sweeping_page); + } + + dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node); + if (gc_compact_heap_cursors_met_p(dest_pool)) { + return dest_pool != heap; + } + } + + return true; +} + +static bool +gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page) +{ + short slot_size = page->slot_size; + + do { + VALUE vp = (VALUE)p; + GC_ASSERT(vp % sizeof(VALUE) == 0); + + if (bitset & 1) { + objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++; + + if (gc_is_moveable_obj(objspace, vp)) { + if (!gc_compact_move(objspace, heap, vp)) { + //the cursors met. bubble up + return false; + } + } + } + p += slot_size; + bitset >>= 1; + } while (bitset); + + return true; +} + +// Iterate up all the objects in page, moving them to where they want to go +static bool +gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) +{ + GC_ASSERT(page == heap->compact_cursor); + + bits_t *mark_bits, *pin_bits; + bits_t bitset; + uintptr_t p = page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + + mark_bits = page->mark_bits; + pin_bits = page->pinned_bits; + + for (int j = 0; j < bitmap_plane_count; j++) { + // objects that can be moved are marked and not pinned + bitset = (mark_bits[j] & ~pin_bits[j]); + if (bitset) { + if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page)) + return false; + } + p += BITS_BITLENGTH * slot_size; + } + + return true; +} + +static bool +gc_compact_all_compacted_p(rb_objspace_t *objspace) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + if (heap->total_pages > 0 && + !gc_compact_heap_cursors_met_p(heap)) { + return false; + } + } + + return true; +} + +static void +gc_sweep_compact(rb_objspace_t *objspace) +{ + gc_compact_start(objspace); +#if RGENGC_CHECK_MODE >= 2 + gc_verify_internal_consistency(objspace); +#endif + + while (!gc_compact_all_compacted_p(objspace)) { + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + if (gc_compact_heap_cursors_met_p(heap)) { + continue; + } + + struct heap_page *start_page = heap->compact_cursor; + + if (!gc_compact_page(objspace, heap, start_page)) { + lock_page_body(objspace, start_page->body); + + continue; + } + + // If we get here, we've finished moving all objects on the compact_cursor page + // So we can lock it and move the cursor on to the next one. + lock_page_body(objspace, start_page->body); + heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node); + } + } + + gc_compact_finish(objspace); + +#if RGENGC_CHECK_MODE >= 2 + gc_verify_internal_consistency(objspace); +#endif +} + +static void +gc_marks_rest(rb_objspace_t *objspace) +{ + gc_report(1, objspace, "gc_marks_rest\n"); + + for (int i = 0; i < HEAP_COUNT; i++) { + (&heaps[i])->pooled_pages = NULL; + } + + if (is_incremental_marking(objspace)) { + while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE); + } + else { + gc_mark_stacked_objects_all(objspace); + } + + gc_marks_finish(objspace); +} + +static bool +gc_marks_step(rb_objspace_t *objspace, size_t slots) +{ + bool marking_finished = false; + + GC_ASSERT(is_marking(objspace)); + if (gc_mark_stacked_objects_incremental(objspace, slots)) { + gc_marks_finish(objspace); + + marking_finished = true; + } + + return marking_finished; +} + +static bool +gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap) +{ + GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD); + bool marking_finished = true; + + gc_marking_enter(objspace); + + if (heap->free_pages) { + gc_report(2, objspace, "gc_marks_continue: has pooled pages"); + + marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots); + } + else { + gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n", + mark_stack_size(&objspace->mark_stack)); + heap->force_incremental_marking_finish_count++; + gc_marks_rest(objspace); + } + + gc_marking_exit(objspace); + + return marking_finished; +} + +static void +gc_marks_start(rb_objspace_t *objspace, int full_mark) +{ + /* start marking */ + gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor"); + gc_mode_transition(objspace, gc_mode_marking); + + if (full_mark) { + size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1; + objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps; + + if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", " + "objspace->rincgc.pooled_page_num: %"PRIdSIZE", " + "objspace->rincgc.step_slots: %"PRIdSIZE", \n", + objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots); + objspace->flags.during_minor_gc = FALSE; + if (ruby_enable_autocompact) { + objspace->flags.during_compacting |= TRUE; + } + objspace->profile.major_gc_count++; + objspace->rgengc.uncollectible_wb_unprotected_objects = 0; + objspace->rgengc.old_objects = 0; + objspace->rgengc.last_major_gc = objspace->profile.count; + objspace->marked_slots = 0; + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + rgengc_mark_and_rememberset_clear(objspace, heap); + heap_move_pooled_pages_to_free_pages(heap); + + if (objspace->flags.during_compacting) { + struct heap_page *page = NULL; + + ccan_list_for_each(&heap->pages, page, page_node) { + page->pinned_slots = 0; + } + } + } + } + else { + objspace->flags.during_minor_gc = TRUE; + objspace->marked_slots = + objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */ + objspace->profile.minor_gc_count++; + + for (int i = 0; i < HEAP_COUNT; i++) { + rgengc_rememberset_mark(objspace, &heaps[i]); + } + } + + mark_roots(objspace, NULL); + + gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n", + full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack)); +} + +static bool +gc_marks(rb_objspace_t *objspace, int full_mark) +{ + gc_marking_enter(objspace); + + bool marking_finished = false; + + /* setup marking */ + + gc_marks_start(objspace, full_mark); + if (!is_incremental_marking(objspace)) { + gc_marks_rest(objspace); + marking_finished = true; + } + +#if RGENGC_PROFILE > 0 + if (gc_prof_record(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->old_objects = objspace->rgengc.old_objects; + } +#endif + + gc_marking_exit(objspace); + + return marking_finished; +} + +/* RGENGC */ + +static void +gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...) +{ + if (level <= RGENGC_DEBUG) { + char buf[1024]; + FILE *out = stderr; + va_list args; + const char *status = " "; + + if (during_gc) { + status = is_full_marking(objspace) ? "+" : "-"; + } + else { + if (is_lazy_sweeping(objspace)) { + status = "S"; + } + if (is_incremental_marking(objspace)) { + status = "M"; + } + } + + va_start(args, fmt); + vsnprintf(buf, 1024, fmt, args); + va_end(args); + + fprintf(out, "%s|", status); + fputs(buf, out); + } +} + +/* bit operations */ + +static int +rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj) +{ + struct heap_page *page = GET_HEAP_PAGE(obj); + bits_t *bits = &page->remembered_bits[0]; + + if (MARKED_IN_BITMAP(bits, obj)) { + return FALSE; + } + else { + page->flags.has_remembered_objects = TRUE; + MARK_IN_BITMAP(bits, obj); + return TRUE; + } +} + +/* wb, etc */ + +/* return FALSE if already remembered */ +static int +rgengc_remember(rb_objspace_t *objspace, VALUE obj) +{ + gc_report(6, objspace, "rgengc_remember: %s %s\n", rb_obj_info(obj), + RVALUE_REMEMBERED(objspace, obj) ? "was already remembered" : "is remembered now"); + + check_rvalue_consistency(objspace, obj); + + if (RGENGC_CHECK_MODE) { + if (RVALUE_WB_UNPROTECTED(objspace, obj)) rb_bug("rgengc_remember: %s is not wb protected.", rb_obj_info(obj)); + } + +#if RGENGC_PROFILE > 0 + if (!RVALUE_REMEMBERED(objspace, obj)) { + if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0) { + objspace->profile.total_remembered_normal_object_count++; +#if RGENGC_PROFILE >= 2 + objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++; +#endif + } + } +#endif /* RGENGC_PROFILE > 0 */ + + return rgengc_remembersetbits_set(objspace, obj); +} + +#ifndef PROFILE_REMEMBERSET_MARK +#define PROFILE_REMEMBERSET_MARK 0 +#endif + +static inline void +rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset, short slot_size) +{ + if (bitset) { + do { + if (bitset & 1) { + VALUE obj = (VALUE)p; + gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj)); + GC_ASSERT(RVALUE_UNCOLLECTIBLE(objspace, obj)); + GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj)); + + gc_mark_children(objspace, obj); + + if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) { + rb_darray_append_without_gc(&objspace->weak_references, obj); + } + } + p += slot_size; + bitset >>= 1; + } while (bitset); + } +} + +static void +rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap) +{ + size_t j; + struct heap_page *page = 0; +#if PROFILE_REMEMBERSET_MARK + int has_old = 0, has_shady = 0, has_both = 0, skip = 0; +#endif + gc_report(1, objspace, "rgengc_rememberset_mark: start\n"); + + ccan_list_for_each(&heap->pages, page, page_node) { + if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) { + uintptr_t p = page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT]; + bits_t *remembered_bits = page->remembered_bits; + bits_t *uncollectible_bits = page->uncollectible_bits; + bits_t *wb_unprotected_bits = page->wb_unprotected_bits; +#if PROFILE_REMEMBERSET_MARK + if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++; + else if (page->flags.has_remembered_objects) has_old++; + else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++; +#endif + for (j=0; j < (size_t)bitmap_plane_count; j++) { + bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]); + remembered_bits[j] = 0; + } + page->flags.has_remembered_objects = FALSE; + + for (j=0; j < (size_t)bitmap_plane_count; j++) { + bitset = bits[j]; + rgengc_rememberset_mark_plane(objspace, p, bitset, slot_size); + p += BITS_BITLENGTH * slot_size; + } + } +#if PROFILE_REMEMBERSET_MARK + else { + skip++; + } +#endif + } + +#if PROFILE_REMEMBERSET_MARK + fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip); +#endif + gc_report(1, objspace, "rgengc_rememberset_mark: finished\n"); +} + +static void +rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap) +{ + struct heap_page *page = 0; + + ccan_list_for_each(&heap->pages, page, page_node) { + memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + page->flags.has_uncollectible_wb_unprotected_objects = FALSE; + page->flags.has_remembered_objects = FALSE; + } +} + +/* RGENGC: APIs */ + +NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)); + +static void +gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace) +{ + if (RGENGC_CHECK_MODE) { + if (!RVALUE_OLD_P(objspace, a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a)); + if ( RVALUE_OLD_P(objspace, b)) rb_bug("gc_writebarrier_generational: %s is an old object.", rb_obj_info(b)); + if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b)); + } + + /* mark `a' and remember (default behavior) */ + if (!RVALUE_REMEMBERED(objspace, a)) { + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + rgengc_remember(objspace, a); + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); + + gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b)); + } + + check_rvalue_consistency(objspace, a); + check_rvalue_consistency(objspace, b); +} + +static void +gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent) +{ + gc_mark_set_parent(objspace, parent); + rgengc_check_relation(objspace, obj); + if (gc_mark_set(objspace, obj) != FALSE) { + gc_aging(objspace, obj); + gc_grey(objspace, obj); + } + gc_mark_set_parent_invalid(objspace); +} + +NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)); + +static void +gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace) +{ + gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, rb_obj_info(b)); + + if (RVALUE_BLACK_P(objspace, a)) { + if (RVALUE_WHITE_P(objspace, b)) { + if (!RVALUE_WB_UNPROTECTED(objspace, a)) { + gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, rb_obj_info(b)); + gc_mark_from(objspace, b, a); + } + } + else if (RVALUE_OLD_P(objspace, a) && !RVALUE_OLD_P(objspace, b)) { + rgengc_remember(objspace, a); + } + + if (RB_UNLIKELY(objspace->flags.during_compacting)) { + MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b); + } + } +} + +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + rb_objspace_t *objspace = objspace_ptr; + +#if RGENGC_CHECK_MODE + if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a); + if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b); +#else + RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(a)); + RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(b)); +#endif + + GC_ASSERT(!during_gc); + GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE); + GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED); + GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE); + GC_ASSERT(RB_BUILTIN_TYPE(b) != T_NONE); + GC_ASSERT(RB_BUILTIN_TYPE(b) != T_MOVED); + GC_ASSERT(RB_BUILTIN_TYPE(b) != T_ZOMBIE); + + retry: + if (!is_incremental_marking(objspace)) { + if (!RVALUE_OLD_P(objspace, a) || RVALUE_OLD_P(objspace, b)) { + // do nothing + } + else { + gc_writebarrier_generational(a, b, objspace); + } + } + else { + bool retry = false; + /* slow path */ + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + if (is_incremental_marking(objspace)) { + gc_writebarrier_incremental(a, b, objspace); + } + else { + retry = true; + } + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); + + if (retry) goto retry; + } + return; +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (RVALUE_WB_UNPROTECTED(objspace, obj)) { + return; + } + else { + gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj), + RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : ""); + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + if (RVALUE_OLD_P(objspace, obj)) { + gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj)); + RVALUE_DEMOTE(objspace, obj); + gc_mark_set(objspace, obj); + gc_remember_unprotected(objspace, obj); + +#if RGENGC_PROFILE + objspace->profile.total_shade_operation_count++; +#if RGENGC_PROFILE >= 2 + objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++; +#endif /* RGENGC_PROFILE >= 2 */ +#endif /* RGENGC_PROFILE */ + } + else { + RVALUE_AGE_RESET(obj); + } + + RB_DEBUG_COUNTER_INC(obj_wb_unprotect); + MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj); + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); + } +} + +void +rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (RVALUE_WB_UNPROTECTED(objspace, obj)) { + rb_gc_impl_writebarrier_unprotect(objspace, dest); + } + rb_gc_impl_copy_finalizer(objspace, dest, obj); +} + +const char * +rb_gc_impl_active_gc_name(void) +{ + return "default"; +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + + gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj)); + + if (is_incremental_marking(objspace) || RVALUE_OLD_P(objspace, obj)) { + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + if (is_incremental_marking(objspace)) { + if (RVALUE_BLACK_P(objspace, obj)) { + gc_grey(objspace, obj); + } + } + else if (RVALUE_OLD_P(objspace, obj)) { + rgengc_remember(objspace, obj); + } + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); + } +} + +struct rb_gc_object_metadata_names { + // Must be ID only + ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking, + ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable; +}; + +#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID)) +static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + size_t n = 0; + static struct rb_gc_object_metadata_names names; + + if (!names.ID_marked) { +#define I(s) names.ID_##s = rb_intern(#s) + I(wb_protected); + I(age); + I(old); + I(uncollectible); + I(marking); + I(marked); + I(pinned); + I(remembered); + I(object_id); + I(shareable); +#undef I + } + +#define SET_ENTRY(na, v) do { \ + GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = names.ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (!RVALUE_WB_UNPROTECTED(objspace, obj)) SET_ENTRY(wb_protected, Qtrue); + SET_ENTRY(age, INT2FIX(RVALUE_AGE_GET(obj))); + if (RVALUE_OLD_P(objspace, obj)) SET_ENTRY(old, Qtrue); + if (RVALUE_UNCOLLECTIBLE(objspace, obj)) SET_ENTRY(uncollectible, Qtrue); + if (RVALUE_MARKING(objspace, obj)) SET_ENTRY(marking, Qtrue); + if (RVALUE_MARKED(objspace, obj)) SET_ENTRY(marked, Qtrue); + if (RVALUE_PINNED(objspace, obj)) SET_ENTRY(pinned, Qtrue); + if (RVALUE_REMEMBERED(objspace, obj)) SET_ENTRY(remembered, Qtrue); + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue); + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; +#undef SET_ENTRY + + return object_metadata_entries; +} + +void * +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) +{ + rb_objspace_t *objspace = objspace_ptr; + + objspace->live_ractor_cache_count++; + + return calloc1(sizeof(rb_ractor_newobj_cache_t)); +} + +void +rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache) +{ + rb_objspace_t *objspace = objspace_ptr; + + objspace->live_ractor_cache_count--; + gc_ractor_newobj_cache_clear(cache, NULL); + free(cache); +} + +static void +heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap) +{ + if (!heap->free_pages) { + if (!heap_page_allocate_and_initialize(objspace, heap)) { + objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE; + heap_page_allocate_and_initialize(objspace, heap); + } + } +} + +static int +ready_to_gc(rb_objspace_t *objspace) +{ + if (dont_gc_val() || during_gc) { + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + heap_ready_to_gc(objspace, heap); + } + return FALSE; + } + else { + return TRUE; + } +} + +static void +gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark) +{ + gc_prof_set_malloc_info(objspace); + { + int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters); + size_t old_limit = malloc_limit; + + /* A net-negative `inc` (more freed than malloc'd since last GC) is + * treated the same as "allocated less than malloc_limit". + * This matches what we were doing pre-monotonic counters, but is it right? */ + if (inc > 0 && (size_t)inc > malloc_limit) { + malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor); + if (malloc_limit > gc_params.malloc_limit_max) { + malloc_limit = gc_params.malloc_limit_max; + } + } + else { + malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */ + if (malloc_limit < gc_params.malloc_limit_min) { + malloc_limit = gc_params.malloc_limit_min; + } + } + + if (0) { + if (old_limit != malloc_limit) { + fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n", + rb_gc_count(), old_limit, malloc_limit); + } + else { + fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n", + rb_gc_count(), malloc_limit); + } + } + } + + /* reset oldmalloc info */ +#if RGENGC_ESTIMATE_OLDMALLOC + if (!full_mark) { + /* Don't snapshot on minor GC: oldmalloc_increase is meant to + * accumulate across minor GCs and only reset at major GC. */ + int64_t oldmalloc_increase = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.oldcounters); + if (oldmalloc_increase > 0 && + (uint64_t)oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) { + gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC; + objspace->rgengc.oldmalloc_increase_limit = + (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor); + + if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) { + objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max; + } + } + + if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRId64"\t%"PRIuSIZE"\t%"PRIdSIZE"\n", + rb_gc_count(), + gc_needs_major_flags, + oldmalloc_increase, + objspace->rgengc.oldmalloc_increase_limit, + gc_params.oldmalloc_limit_max); + } + else { + if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) { + objspace->rgengc.oldmalloc_increase_limit = + (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1)); + if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) { + objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min; + } + } + } +#endif +} + +static int +garbage_collect(rb_objspace_t *objspace, unsigned int reason) +{ + int ret; + + int lev = RB_GC_VM_LOCK(); + { +#if GC_PROFILE_MORE_DETAIL + objspace->profile.prepare_time = getrusage_time(); +#endif + + gc_rest(objspace); + +#if GC_PROFILE_MORE_DETAIL + objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time; +#endif + + ret = gc_start(objspace, reason); + } + RB_GC_VM_UNLOCK(lev); + + return ret; +} + +static int +gc_start(rb_objspace_t *objspace, unsigned int reason) +{ + unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK); + + if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */ + if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */ + + rb_gc_initialize_vm_context(&objspace->vm_context); + + GC_ASSERT(gc_mode(objspace) == gc_mode_none, "gc_mode is %s\n", gc_mode_name(gc_mode(objspace))); + GC_ASSERT(!is_lazy_sweeping(objspace)); + GC_ASSERT(!is_incremental_marking(objspace)); + + unsigned int lock_lev; + gc_enter(objspace, gc_enter_event_start, &lock_lev); + + /* reason may be clobbered, later, so keep set immediate_sweep here */ + objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); + +#if RGENGC_CHECK_MODE >= 2 + gc_verify_internal_consistency(objspace); +#endif + + if (ruby_gc_stressful) { + int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0; + + if ((flag & (1 << gc_stress_no_major)) == 0) { + do_full_mark = TRUE; + } + + objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep)); + } + + if (gc_needs_major_flags) { + reason |= gc_needs_major_flags; + do_full_mark = TRUE; + } + + /* if major gc has been disabled, never do a full mark */ + if (!gc_config_full_mark_val) { + do_full_mark = FALSE; + } + gc_needs_major_flags = GPR_FLAG_NONE; + + if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) { + reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */ + } + + if (objspace->flags.dont_incremental || + reason & GPR_FLAG_IMMEDIATE_MARK || + ruby_gc_stressful) { + objspace->flags.during_incremental_marking = FALSE; + } + else { + objspace->flags.during_incremental_marking = do_full_mark; + } + + /* Explicitly enable compaction (GC.compact) */ + if (do_full_mark && ruby_enable_autocompact) { + objspace->flags.during_compacting = TRUE; +#if RGENGC_CHECK_MODE + objspace->rcompactor.compare_func = ruby_autocompact_compare_func; +#endif + } + else { + objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT); + } + + if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) { + objspace->flags.immediate_sweep = TRUE; + } + + if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP; + + gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n", + reason, + do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep); + + RB_DEBUG_COUNTER_INC(gc_count); + + if (reason & GPR_FLAG_MAJOR_MASK) { + (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE); + (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN); + (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY); + (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE); +#if RGENGC_ESTIMATE_OLDMALLOC + (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC); +#endif + } + else { + (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ); + (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC); + (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD); + (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI); + (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS); + } + + objspace->profile.count++; + objspace->profile.latest_gc_info = reason; + objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace); + objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted); + objspace->profile.heap_total_slots_at_gc_start = objspace_available_slots(objspace); + objspace->profile.weak_references_count = 0; + gc_prof_setup_new_record(objspace, reason); + gc_reset_malloc_info(objspace, do_full_mark); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START); + + GC_ASSERT(during_gc); + + gc_prof_timer_start(objspace); + { + if (gc_marks(objspace, do_full_mark)) { + gc_sweep(objspace); + } + } + gc_prof_timer_stop(objspace); + + gc_exit(objspace, gc_enter_event_start, &lock_lev); + return TRUE; +} + +static void +gc_rest(rb_objspace_t *objspace) +{ + if (is_incremental_marking(objspace) || is_lazy_sweeping(objspace)) { + unsigned int lock_lev; + gc_enter(objspace, gc_enter_event_rest, &lock_lev); + + if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace); + + if (is_incremental_marking(objspace)) { + gc_marking_enter(objspace); + gc_marks_rest(objspace); + gc_marking_exit(objspace); + + gc_sweep(objspace); + } + + if (is_lazy_sweeping(objspace)) { + gc_sweeping_enter(objspace); + gc_sweep_rest(objspace); + gc_sweeping_exit(objspace); + } + + gc_exit(objspace, gc_enter_event_rest, &lock_lev); + } +} + +struct objspace_and_reason { + rb_objspace_t *objspace; + unsigned int reason; +}; + +static void +gc_current_status_fill(rb_objspace_t *objspace, char *buff) +{ + int i = 0; + if (is_marking(objspace)) { + buff[i++] = 'M'; + if (is_full_marking(objspace)) buff[i++] = 'F'; + if (is_incremental_marking(objspace)) buff[i++] = 'I'; + } + else if (is_sweeping(objspace)) { + buff[i++] = 'S'; + if (is_lazy_sweeping(objspace)) buff[i++] = 'L'; + } + else { + buff[i++] = 'N'; + } + buff[i] = '\0'; +} + +static const char * +gc_current_status(rb_objspace_t *objspace) +{ + static char buff[0x10]; + gc_current_status_fill(objspace, buff); + return buff; +} + +#if PRINT_ENTER_EXIT_TICK + +static tick_t last_exit_tick; +static tick_t enter_tick; +static int enter_count = 0; +static char last_gc_status[0x10]; + +static inline void +gc_record(rb_objspace_t *objspace, int direction, const char *event) +{ + if (direction == 0) { /* enter */ + enter_count++; + enter_tick = tick(); + gc_current_status_fill(objspace, last_gc_status); + } + else { /* exit */ + tick_t exit_tick = tick(); + char current_gc_status[0x10]; + gc_current_status_fill(objspace, current_gc_status); +#if 1 + /* [last mutator time] [gc time] [event] */ + fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n", + enter_tick - last_exit_tick, + exit_tick - enter_tick, + event, + last_gc_status, current_gc_status, + (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-'); + last_exit_tick = exit_tick; +#else + /* [enter_tick] [gc time] [event] */ + fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n", + enter_tick, + exit_tick - enter_tick, + event, + last_gc_status, current_gc_status, + (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-'); +#endif + } +} +#else /* PRINT_ENTER_EXIT_TICK */ +static inline void +gc_record(rb_objspace_t *objspace, int direction, const char *event) +{ + /* null */ +} +#endif /* PRINT_ENTER_EXIT_TICK */ + +static const char * +gc_enter_event_cstr(enum gc_enter_event event) +{ + switch (event) { + case gc_enter_event_start: return "start"; + case gc_enter_event_continue: return "continue"; + case gc_enter_event_rest: return "rest"; + case gc_enter_event_finalizer: return "finalizer"; + } + return NULL; +} + +static void +gc_enter_count(enum gc_enter_event event) +{ + switch (event) { + case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start); break; + case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue); break; + case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest); break; + case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break; + } +} + +static bool current_process_time(struct timespec *ts); + +static void +gc_clock_start(struct timespec *ts) +{ + if (!current_process_time(ts)) { + ts->tv_sec = 0; + ts->tv_nsec = 0; + } +} + +static unsigned long long +gc_clock_end(struct timespec *ts) +{ + struct timespec end_time; + + if ((ts->tv_sec > 0 || ts->tv_nsec > 0) && + current_process_time(&end_time) && + end_time.tv_sec >= ts->tv_sec) { + return (unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) + + (end_time.tv_nsec - ts->tv_nsec); + } + + return 0; +} + +static inline void +gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev) +{ + *lock_lev = RB_GC_VM_LOCK(); + + switch (event) { + case gc_enter_event_rest: + case gc_enter_event_start: + case gc_enter_event_continue: + // stop other ractors + rb_gc_vm_barrier(); + break; + default: + break; + } + + gc_enter_count(event); + if (RB_UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0"); + if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace); + + during_gc = TRUE; + RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace)); + gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace)); + gc_record(objspace, 0, gc_enter_event_cstr(event)); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_ENTER); +} + +static inline void +gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev) +{ + GC_ASSERT(during_gc != 0); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_EXIT); + + gc_record(objspace, 1, gc_enter_event_cstr(event)); + RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace)); + gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace)); + during_gc = FALSE; + + RB_GC_VM_UNLOCK(*lock_lev); +} + +#ifndef MEASURE_GC +#define MEASURE_GC (objspace->flags.measure_gc) +#endif + +static void +gc_marking_enter(rb_objspace_t *objspace) +{ + GC_ASSERT(during_gc != 0); + + gc_prof_mark_timer_start(objspace); + + if (MEASURE_GC) { + gc_clock_start(&objspace->profile.marking_start_time); + } + + rb_gc_initialize_vm_context(&objspace->vm_context); +} + +static void +gc_marking_exit(rb_objspace_t *objspace) +{ + GC_ASSERT(during_gc != 0); + + if (MEASURE_GC) { + objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time); + } + + gc_prof_mark_timer_stop(objspace); +} + +static void +gc_sweeping_enter(rb_objspace_t *objspace) +{ + GC_ASSERT(during_gc != 0); + + if (MEASURE_GC) { + gc_clock_start(&objspace->profile.sweeping_start_time); + } +} + +static void +gc_sweeping_exit(rb_objspace_t *objspace) +{ + GC_ASSERT(during_gc != 0); + + if (MEASURE_GC) { + objspace->profile.sweeping_time_ns += gc_clock_end(&objspace->profile.sweeping_start_time); + } +} + +static void * +gc_with_gvl(void *ptr) +{ + struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr; + return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason); +} + +int ruby_thread_has_gvl_p(void); + +static int +garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason) +{ + if (dont_gc_val()) { + return TRUE; + } + else if (!ruby_native_thread_p()) { + return TRUE; + } + else if (!ruby_thread_has_gvl_p()) { + void *ret; + struct objspace_and_reason oar; + oar.objspace = objspace; + oar.reason = reason; + ret = rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar); + + return !!ret; + } + else { + return garbage_collect(objspace, reason); + } +} + +static int +gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data) +{ + rb_objspace_t *objspace = (rb_objspace_t *)data; + + VALUE v = (VALUE)vstart; + for (; v != (VALUE)vend; v += stride) { + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_ZOMBIE: + break; + default: + rb_gc_prepare_heap_process_object(v); + if (!RVALUE_OLD_P(objspace, v) && !RVALUE_WB_UNPROTECTED(objspace, v)) { + RVALUE_AGE_SET_CANDIDATE(objspace, v); + } + } + } + } + + return 0; +} + +void +rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact) +{ + rb_objspace_t *objspace = objspace_ptr; + unsigned int reason = (GPR_FLAG_FULL_MARK | + GPR_FLAG_IMMEDIATE_MARK | + GPR_FLAG_IMMEDIATE_SWEEP | + GPR_FLAG_METHOD); + + int full_marking_p = gc_config_full_mark_val; + gc_config_full_mark_set(TRUE); + + /* For now, compact implies full mark / sweep, so ignore other flags */ + if (compact) { + GC_ASSERT(GC_COMPACTION_SUPPORTED); + + reason |= GPR_FLAG_COMPACT; + } + else { + if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK; + if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK; + if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP; + } + + garbage_collect(objspace, reason); + gc_finalize_deferred(objspace); + + gc_config_full_mark_set(full_marking_p); +} + +void +rb_gc_impl_prepare_heap(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + size_t orig_total_slots = objspace_available_slots(objspace); + size_t orig_allocatable_bytes = objspace->heap_pages.allocatable_bytes; + + rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr); + + double orig_max_free_slots = gc_params.heap_free_slots_max_ratio; + /* Ensure that all empty pages are moved onto empty_pages. */ + gc_params.heap_free_slots_max_ratio = 0.0; + rb_gc_impl_start(objspace, true, true, true, true); + gc_params.heap_free_slots_max_ratio = orig_max_free_slots; + + objspace->heap_pages.allocatable_bytes = 0; + heap_pages_freeable_pages = objspace->empty_pages_count; + heap_pages_free_unused_pages(objspace_ptr); + GC_ASSERT(heap_pages_freeable_pages == 0); + GC_ASSERT(objspace->empty_pages_count == 0); + objspace->heap_pages.allocatable_bytes = orig_allocatable_bytes; + + size_t total_slots = objspace_available_slots(objspace); + if (orig_total_slots > total_slots) { + objspace->heap_pages.allocatable_bytes += (orig_total_slots - total_slots) * heaps[0].slot_size; + } + +#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER) + malloc_trim(0); +#endif +} + +static int +gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj) +{ + GC_ASSERT(!SPECIAL_CONST_P(obj)); + + switch (BUILTIN_TYPE(obj)) { + case T_NONE: + case T_MOVED: + case T_ZOMBIE: + return FALSE; + case T_SYMBOL: + case T_STRING: + case T_OBJECT: + case T_FLOAT: + case T_IMEMO: + case T_ARRAY: + case T_BIGNUM: + case T_ICLASS: + case T_MODULE: + case T_REGEXP: + case T_DATA: + case T_MATCH: + case T_STRUCT: + case T_HASH: + case T_FILE: + case T_COMPLEX: + case T_RATIONAL: + case T_NODE: + case T_CLASS: + if (FL_TEST_RAW(obj, FL_FINALIZE)) { + /* The finalizer table is a numtable. It looks up objects by address. + * We can't mark the keys in the finalizer table because that would + * prevent the objects from being collected. This check prevents + * objects that are keys in the finalizer table from being moved + * without directly pinning them. */ + GC_ASSERT(st_is_member(finalizer_table, obj)); + + return FALSE; + } + GC_ASSERT(RVALUE_MARKED(objspace, obj)); + GC_ASSERT(!RVALUE_PINNED(objspace, obj)); + + return TRUE; + + default: + rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj)); + break; + } + + return FALSE; +} + +void rb_mv_generic_ivar(VALUE src, VALUE dst); + +static VALUE +gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_page, struct heap_page *dest_page) +{ + size_t src_slot_size = src_page->slot_size; + size_t slot_size = dest_page->slot_size; + + int marked; + int wb_unprotected; + int uncollectible; + int age; + + gc_report(4, objspace, "Moving object: %p -> %p\n", (void *)src, (void *)dest); + + GC_ASSERT(BUILTIN_TYPE(src) != T_NONE); + GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest)); + + GC_ASSERT(!RVALUE_MARKING(objspace, src)); + + /* Save off bits for current object. */ + marked = RVALUE_MARKED(objspace, src); + wb_unprotected = RVALUE_WB_UNPROTECTED(objspace, src); + uncollectible = RVALUE_UNCOLLECTIBLE(objspace, src); + bool remembered = RVALUE_REMEMBERED(objspace, src); + age = RVALUE_AGE_GET(src); + + /* Clear bits for eventual T_MOVED */ + CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src); + CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src); + CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src); + CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src); + + /* Move the object */ + memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size)); + + if (src_slot_size != slot_size && RB_TYPE_P(src, T_OBJECT)) { + rb_gc_obj_changed_pool(dest, dest_page->heap - heaps); + } + + if (RVALUE_OVERHEAD > 0) { + void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD); + void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD); + + memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD); + } + + memset((void *)src, 0, src_slot_size); + RVALUE_AGE_SET_BITMAP(src, 0); + + /* Set bits for object in new location */ + if (remembered) { + MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest); + } + else { + CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest); + } + + if (marked) { + MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest); + } + else { + CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest); + } + + if (wb_unprotected) { + MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest); + } + else { + CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest); + } + + if (uncollectible) { + MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest); + } + else { + CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest); + } + + RVALUE_AGE_SET(dest, age); + /* Assign forwarding address */ + RMOVED(src)->flags = T_MOVED; + RMOVED(src)->dummy = Qundef; + RMOVED(src)->destination = dest; + GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE); + + GET_HEAP_PAGE(src)->heap->total_freed_objects++; + GET_HEAP_PAGE(dest)->heap->total_allocated_objects++; + + return src; +} + +#if GC_CAN_COMPILE_COMPACTION +static int +compare_pinned_slots(const void *left, const void *right, void *dummy) +{ + struct heap_page *left_page; + struct heap_page *right_page; + + left_page = *(struct heap_page * const *)left; + right_page = *(struct heap_page * const *)right; + + return left_page->pinned_slots - right_page->pinned_slots; +} + +static int +compare_free_slots(const void *left, const void *right, void *dummy) +{ + struct heap_page *left_page; + struct heap_page *right_page; + + left_page = *(struct heap_page * const *)left; + right_page = *(struct heap_page * const *)right; + + return left_page->free_slots - right_page->free_slots; +} + +static void +gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func) +{ + for (int j = 0; j < HEAP_COUNT; j++) { + rb_heap_t *heap = &heaps[j]; + + size_t total_pages = heap->total_pages; + size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError); + struct heap_page *page = 0, **page_list = malloc(size); + size_t i = 0; + + heap->free_pages = NULL; + ccan_list_for_each(&heap->pages, page, page_node) { + page_list[i++] = page; + GC_ASSERT(page); + } + + GC_ASSERT((size_t)i == total_pages); + + /* Sort the heap so "filled pages" are first. `heap_add_page` adds to the + * head of the list, so empty pages will end up at the start of the heap */ + ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL); + + /* Reset the eden heap */ + ccan_list_head_init(&heap->pages); + + for (i = 0; i < total_pages; i++) { + ccan_list_add(&heap->pages, &page_list[i]->page_node); + if (page_list[i]->free_slots != 0) { + heap_add_freepage(heap, page_list[i]); + } + } + + free(page_list); + } +} +#endif + +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + /* no-op */ +} + +bool +rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) +{ + return gc_object_moved_p(objspace_ptr, obj); +} + +static int +gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t *objspace, struct heap_page *page) +{ + VALUE v = (VALUE)vstart; + + page->flags.has_uncollectible_wb_unprotected_objects = FALSE; + page->flags.has_remembered_objects = FALSE; + + /* For each object on the page */ + for (; v != (VALUE)vend; v += stride) { + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_MOVED: + case T_ZOMBIE: + break; + default: + if (RVALUE_WB_UNPROTECTED(objspace, v)) { + page->flags.has_uncollectible_wb_unprotected_objects = TRUE; + } + if (RVALUE_REMEMBERED(objspace, v)) { + page->flags.has_remembered_objects = TRUE; + } + if (page->flags.before_sweep) { + if (RVALUE_MARKED(objspace, v)) { + rb_gc_update_object_references(objspace, v); + } + } + else { + rb_gc_update_object_references(objspace, v); + } + } + } + } + + return 0; +} + +static int +gc_update_references_weak_table_i(VALUE obj, void *data) +{ + int ret; + asan_unpoisoning_object(obj) { + ret = BUILTIN_TYPE(obj) == T_MOVED ? ST_REPLACE : ST_CONTINUE; + } + return ret; +} + +static int +gc_update_references_weak_table_replace_i(VALUE *obj, void *data) +{ + *obj = rb_gc_location(*obj); + + return ST_CONTINUE; +} + +static void +gc_update_references(rb_objspace_t *objspace) +{ + objspace->flags.during_reference_updating = true; + + rb_gc_before_updating_jit_code(); + + struct heap_page *page = NULL; + + for (int i = 0; i < HEAP_COUNT; i++) { + bool should_set_mark_bits = TRUE; + rb_heap_t *heap = &heaps[i]; + + ccan_list_for_each(&heap->pages, page, page_node) { + uintptr_t start = (uintptr_t)page->start; + uintptr_t end = start + (page->total_slots * heap->slot_size); + + gc_ref_update((void *)start, (void *)end, heap->slot_size, objspace, page); + if (page == heap->sweeping_page) { + should_set_mark_bits = FALSE; + } + if (should_set_mark_bits) { + gc_setup_mark_bits(page); + } + } + } + + gc_update_table_refs(finalizer_table); + + rb_gc_update_vm_references((void *)objspace); + + for (int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) { + rb_gc_vm_weak_table_foreach( + gc_update_references_weak_table_i, + gc_update_references_weak_table_replace_i, + NULL, + false, + table + ); + } + + rb_gc_after_updating_jit_code(); + + objspace->flags.during_reference_updating = false; +} + +#if GC_CAN_COMPILE_COMPACTION +static void +root_obj_check_moved_i(const char *category, VALUE obj, void *data) +{ + rb_objspace_t *objspace = data; + + if (gc_object_moved_p(objspace, obj)) { + rb_bug("ROOT %s points to MOVED: %p -> %s", category, (void *)obj, rb_obj_info(rb_gc_impl_location(objspace, obj))); + } +} + +static void +reachable_object_check_moved_i(VALUE ref, void *data) +{ + VALUE parent = (VALUE)data; + if (gc_object_moved_p(rb_gc_get_objspace(), ref)) { + rb_bug("Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref))); + } +} + +static int +heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data) +{ + rb_objspace_t *objspace = data; + + VALUE v = (VALUE)vstart; + for (; v != (VALUE)vend; v += stride) { + if (gc_object_moved_p(objspace, v)) { + /* Moved object still on the heap, something may have a reference. */ + } + else { + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_ZOMBIE: + break; + default: + if (!rb_gc_impl_garbage_object_p(objspace, v)) { + rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v); + } + } + } + } + } + + return 0; +} +#endif + +bool +rb_gc_impl_during_gc_p(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + return during_gc; +} + +#if RGENGC_PROFILE >= 2 + +static const char* +type_name(int type, VALUE obj) +{ + switch ((enum ruby_value_type)type) { + case RUBY_T_NONE: return "T_NONE"; + case RUBY_T_OBJECT: return "T_OBJECT"; + case RUBY_T_CLASS: return "T_CLASS"; + case RUBY_T_MODULE: return "T_MODULE"; + case RUBY_T_FLOAT: return "T_FLOAT"; + case RUBY_T_STRING: return "T_STRING"; + case RUBY_T_REGEXP: return "T_REGEXP"; + case RUBY_T_ARRAY: return "T_ARRAY"; + case RUBY_T_HASH: return "T_HASH"; + case RUBY_T_STRUCT: return "T_STRUCT"; + case RUBY_T_BIGNUM: return "T_BIGNUM"; + case RUBY_T_FILE: return "T_FILE"; + case RUBY_T_DATA: return "T_DATA"; + case RUBY_T_MATCH: return "T_MATCH"; + case RUBY_T_COMPLEX: return "T_COMPLEX"; + case RUBY_T_RATIONAL: return "T_RATIONAL"; + case RUBY_T_NIL: return "T_NIL"; + case RUBY_T_TRUE: return "T_TRUE"; + case RUBY_T_FALSE: return "T_FALSE"; + case RUBY_T_SYMBOL: return "T_SYMBOL"; + case RUBY_T_FIXNUM: return "T_FIXNUM"; + case RUBY_T_UNDEF: return "T_UNDEF"; + case RUBY_T_IMEMO: return "T_IMEMO"; + case RUBY_T_NODE: return "T_NODE"; + case RUBY_T_ICLASS: return "T_ICLASS"; + case RUBY_T_ZOMBIE: return "T_ZOMBIE"; + case RUBY_T_MOVED: return "T_MOVED"; + default: return "unknown"; + } +} + +static void +gc_count_add_each_types(VALUE hash, const char *name, const size_t *types) +{ + VALUE result = rb_hash_new_with_size(T_MASK); + int i; + for (i=0; i<T_MASK; i++) { + const char *type = type_name(i, 0); + rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i])); + } + rb_hash_aset(hash, ID2SYM(rb_intern(name)), result); +} +#endif + +size_t +rb_gc_impl_gc_count(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + return objspace->profile.count; +} + +static VALUE +gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags) +{ + static VALUE sym_major_by = Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by; + static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress; +#if RGENGC_ESTIMATE_OLDMALLOC + static VALUE sym_oldmalloc; +#endif + static VALUE sym_newobj, sym_malloc, sym_method, sym_capi; + static VALUE sym_none, sym_marking, sym_sweeping; + static VALUE sym_weak_references_count; + VALUE hash = Qnil, key = Qnil; + VALUE major_by, need_major_by; + unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info; + + if (SYMBOL_P(hash_or_key)) { + key = hash_or_key; + } + else if (RB_TYPE_P(hash_or_key, T_HASH)) { + hash = hash_or_key; + } + else { + rb_bug("gc_info_decode: non-hash or symbol given"); + } + + if (NIL_P(sym_major_by)) { +#define S(s) sym_##s = ID2SYM(rb_intern_const(#s)) + S(major_by); + S(gc_by); + S(immediate_sweep); + S(have_finalizer); + S(state); + S(need_major_by); + + S(stress); + S(nofree); + S(oldgen); + S(shady); + S(force); +#if RGENGC_ESTIMATE_OLDMALLOC + S(oldmalloc); +#endif + S(newobj); + S(malloc); + S(method); + S(capi); + + S(none); + S(marking); + S(sweeping); + + S(weak_references_count); +#undef S + } + +#define SET(name, attr) \ + if (key == sym_##name) \ + return (attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, sym_##name, (attr)); + + major_by = + (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree : + (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen : + (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady : + (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force : +#if RGENGC_ESTIMATE_OLDMALLOC + (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc : +#endif + Qnil; + SET(major_by, major_by); + + if (orig_flags == 0) { /* set need_major_by only if flags not set explicitly */ + unsigned int need_major_flags = gc_needs_major_flags; + need_major_by = + (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree : + (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen : + (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady : + (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force : +#if RGENGC_ESTIMATE_OLDMALLOC + (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc : +#endif + Qnil; + SET(need_major_by, need_major_by); + } + + SET(gc_by, + (flags & GPR_FLAG_NEWOBJ) ? sym_newobj : + (flags & GPR_FLAG_MALLOC) ? sym_malloc : + (flags & GPR_FLAG_METHOD) ? sym_method : + (flags & GPR_FLAG_CAPI) ? sym_capi : + (flags & GPR_FLAG_STRESS) ? sym_stress : + Qnil + ); + + SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse); + SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ? Qtrue : Qfalse); + + if (orig_flags == 0) { + SET(state, gc_mode(objspace) == gc_mode_none ? sym_none : + gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping); + } + + SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count)); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +VALUE +rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key) +{ + rb_objspace_t *objspace = objspace_ptr; + + return gc_info_decode(objspace, key, 0); +} + + +enum gc_stat_sym { + gc_stat_sym_count, + gc_stat_sym_time, + gc_stat_sym_marking_time, + gc_stat_sym_sweeping_time, + gc_stat_sym_heap_allocated_pages, + gc_stat_sym_heap_empty_pages, + gc_stat_sym_heap_allocatable_bytes, + gc_stat_sym_heap_available_slots, + gc_stat_sym_heap_live_slots, + gc_stat_sym_heap_free_slots, + gc_stat_sym_heap_final_slots, + gc_stat_sym_heap_marked_slots, + gc_stat_sym_heap_eden_pages, + gc_stat_sym_total_allocated_pages, + gc_stat_sym_total_freed_pages, + gc_stat_sym_total_allocated_objects, + gc_stat_sym_total_freed_objects, + gc_stat_sym_total_malloc_bytes, + gc_stat_sym_total_free_bytes, + gc_stat_sym_malloc_increase_bytes, + gc_stat_sym_malloc_increase_bytes_limit, + gc_stat_sym_minor_gc_count, + gc_stat_sym_major_gc_count, + gc_stat_sym_compact_count, + gc_stat_sym_read_barrier_faults, + gc_stat_sym_total_moved_objects, + gc_stat_sym_remembered_wb_unprotected_objects, + gc_stat_sym_remembered_wb_unprotected_objects_limit, + gc_stat_sym_old_objects, + gc_stat_sym_old_objects_limit, +#if RGENGC_ESTIMATE_OLDMALLOC + gc_stat_sym_oldmalloc_increase_bytes, + gc_stat_sym_oldmalloc_increase_bytes_limit, +#endif +#if RGENGC_PROFILE + gc_stat_sym_total_generated_normal_object_count, + gc_stat_sym_total_generated_shady_object_count, + gc_stat_sym_total_shade_operation_count, + gc_stat_sym_total_promoted_count, + gc_stat_sym_total_remembered_normal_object_count, + gc_stat_sym_total_remembered_shady_object_count, +#endif + gc_stat_sym_last +}; + +static VALUE gc_stat_symbols[gc_stat_sym_last]; + +static void +setup_gc_stat_symbols(void) +{ + if (gc_stat_symbols[0] == 0) { +#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s)) + S(count); + S(time); + S(marking_time), + S(sweeping_time), + S(heap_allocated_pages); + S(heap_empty_pages); + S(heap_allocatable_bytes); + S(heap_available_slots); + S(heap_live_slots); + S(heap_free_slots); + S(heap_final_slots); + S(heap_marked_slots); + S(heap_eden_pages); + S(total_allocated_pages); + S(total_freed_pages); + S(total_allocated_objects); + S(total_freed_objects); + S(total_malloc_bytes); + S(total_free_bytes); + S(malloc_increase_bytes); + S(malloc_increase_bytes_limit); + S(minor_gc_count); + S(major_gc_count); + S(compact_count); + S(read_barrier_faults); + S(total_moved_objects); + S(remembered_wb_unprotected_objects); + S(remembered_wb_unprotected_objects_limit); + S(old_objects); + S(old_objects_limit); +#if RGENGC_ESTIMATE_OLDMALLOC + S(oldmalloc_increase_bytes); + S(oldmalloc_increase_bytes_limit); +#endif +#if RGENGC_PROFILE + S(total_generated_normal_object_count); + S(total_generated_shady_object_count); + S(total_shade_operation_count); + S(total_promoted_count); + S(total_remembered_normal_object_count); + S(total_remembered_shady_object_count); +#endif /* RGENGC_PROFILE */ +#undef S + } +} + +static uint64_t +ns_to_ms(uint64_t ns) +{ + return ns / (1000 * 1000); +} + +static void malloc_increase_local_flush(rb_objspace_t *objspace); + +VALUE +rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) +{ + rb_objspace_t *objspace = objspace_ptr; + VALUE hash = Qnil, key = Qnil; + + setup_gc_stat_symbols(); + + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + malloc_increase_local_flush(objspace); + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + hash = hash_or_sym; + } + else if (SYMBOL_P(hash_or_sym)) { + key = hash_or_sym; + } + else { + rb_bug("non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == gc_stat_symbols[gc_stat_sym_##name]) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr)); +#define SET64(name, attr) \ + if (key == gc_stat_symbols[gc_stat_sym_##name]) \ + return ULL2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr)); + + SET(count, objspace->profile.count); + SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM + SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns)); + SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns)); + + { + uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc); + uint64_t total_free = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.free); + SET64(total_malloc_bytes, total_malloc); + SET64(total_free_bytes, total_free); + } + + /* implementation dependent counters (small / fixnum-safe) */ + SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted)); + SET(heap_empty_pages, objspace->empty_pages_count) + SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes); + SET(heap_eden_pages, heap_eden_total_pages(objspace)); + SET(total_allocated_pages, objspace->heap_pages.allocated_pages); + SET(total_freed_pages, objspace->heap_pages.freed_pages); + SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)); + SET(malloc_increase_bytes_limit, malloc_limit); + SET(minor_gc_count, objspace->profile.minor_gc_count); + SET(major_gc_count, objspace->profile.major_gc_count); + SET(compact_count, objspace->profile.compact_count); + SET(read_barrier_faults, objspace->profile.read_barrier_faults); + SET(total_moved_objects, objspace->rcompactor.total_moved); + SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects); + SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit); + SET(old_objects, objspace->rgengc.old_objects); + SET(old_objects_limit, objspace->rgengc.old_objects_limit); +#if RGENGC_ESTIMATE_OLDMALLOC + SET(oldmalloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.oldcounters)); + SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit); +#endif + + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + SET(total_allocated_objects, total_allocated_objects(objspace)); + SET(total_freed_objects, total_freed_objects(objspace)); + SET(heap_available_slots, objspace_available_slots(objspace)); + SET(heap_live_slots, objspace_live_slots(objspace)); + SET(heap_free_slots, objspace_free_slots(objspace)); + SET(heap_final_slots, total_final_slots_count(objspace)); + SET(heap_marked_slots, objspace->marked_slots); + +#if RGENGC_PROFILE + SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count); + SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count); + SET(total_shade_operation_count, objspace->profile.total_shade_operation_count); + SET(total_promoted_count, objspace->profile.total_promoted_count); + SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count); + SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count); +#endif /* RGENGC_PROFILE */ +#undef SET +#undef SET64 + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + +#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2 + if (hash != Qnil) { + gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types); + gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types); + gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types); + gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types); + gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types); + gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types); + } +#endif + + return hash; +} + +enum gc_stat_heap_sym { + gc_stat_heap_sym_slot_size, + gc_stat_heap_sym_heap_live_slots, + gc_stat_heap_sym_heap_free_slots, + gc_stat_heap_sym_heap_final_slots, + gc_stat_heap_sym_heap_eden_pages, + gc_stat_heap_sym_heap_eden_slots, + gc_stat_heap_sym_total_allocated_pages, + gc_stat_heap_sym_force_major_gc_count, + gc_stat_heap_sym_force_incremental_marking_finish_count, + gc_stat_heap_sym_heap_allocatable_slots, + gc_stat_heap_sym_total_allocated_objects, + gc_stat_heap_sym_total_freed_objects, + gc_stat_heap_sym_last +}; + +static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last]; + +static void +setup_gc_stat_heap_symbols(void) +{ + if (gc_stat_heap_symbols[0] == 0) { +#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s)) + S(slot_size); + S(heap_live_slots); + S(heap_free_slots); + S(heap_final_slots); + S(heap_eden_pages); + S(heap_eden_slots); + S(heap_allocatable_slots); + S(total_allocated_pages); + S(force_major_gc_count); + S(force_incremental_marking_finish_count); + S(total_allocated_objects); + S(total_freed_objects); +#undef S + } +} + +static VALUE +stat_one_heap(rb_objspace_t *objspace, rb_heap_t *heap, VALUE hash, VALUE key) +{ +#define SET(name, attr) \ + if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr)); + + SET(slot_size, heap->slot_size); + SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count); + SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects)); + SET(heap_final_slots, heap->final_slots_count); + SET(heap_eden_pages, heap->total_pages); + SET(heap_eden_slots, heap->total_slots); + SET(heap_allocatable_slots, objspace->heap_pages.allocatable_bytes / heap->slot_size); + SET(total_allocated_pages, heap->total_allocated_pages); + SET(force_major_gc_count, heap->force_major_gc_count); + SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count); + SET(total_allocated_objects, heap->total_allocated_objects); + SET(total_freed_objects, heap->total_freed_objects); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +VALUE +rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) +{ + rb_objspace_t *objspace = objspace_ptr; + + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + + setup_gc_stat_heap_symbols(); + + if (NIL_P(heap_name)) { + if (!RB_TYPE_P(hash_or_sym, T_HASH)) { + rb_bug("non-hash given"); + } + + for (int i = 0; i < HEAP_COUNT; i++) { + VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i)); + if (NIL_P(hash)) { + hash = rb_hash_new(); + rb_hash_aset(hash_or_sym, INT2FIX(i), hash); + } + + stat_one_heap(objspace, &heaps[i], hash, Qnil); + } + } + else if (FIXNUM_P(heap_name)) { + int heap_idx = FIX2INT(heap_name); + + if (heap_idx < 0 || heap_idx >= HEAP_COUNT) { + rb_raise(rb_eArgError, "size pool index out of range"); + } + + if (SYMBOL_P(hash_or_sym)) { + return stat_one_heap(objspace, &heaps[heap_idx], Qnil, hash_or_sym); + } + else if (RB_TYPE_P(hash_or_sym, T_HASH)) { + return stat_one_heap(objspace, &heaps[heap_idx], hash_or_sym, Qnil); + } + else { + rb_bug("non-hash or symbol given"); + } + } + else { + rb_bug("heap_name must be nil or an Integer"); + } + + return hash_or_sym; +} + +/* I could include internal.h for this, but doing so undefines some Array macros + * necessary for initialising objects, and I don't want to include all the array + * headers to get them back + * TODO: Investigate why RARRAY_AREF gets undefined in internal.h + */ +#ifndef RBOOL +#define RBOOL(v) (v ? Qtrue : Qfalse) +#endif + +VALUE +rb_gc_impl_config_get(void *objspace_ptr) +{ +#define sym(name) ID2SYM(rb_intern_const(name)) + rb_objspace_t *objspace = objspace_ptr; + VALUE hash = rb_hash_new(); + + rb_hash_aset(hash, sym("rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val)); + + return hash; +} + +static int +gc_config_set_key(VALUE key, VALUE value, VALUE data) +{ + rb_objspace_t *objspace = (rb_objspace_t *)data; + if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) { + gc_rest(objspace); + gc_config_full_mark_set(RTEST(value)); + } + return ST_CONTINUE; +} + +void +rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (!RB_TYPE_P(hash, T_HASH)) { + rb_raise(rb_eArgError, "expected keyword arguments"); + } + + rb_hash_foreach(hash, gc_config_set_key, (st_data_t)objspace); +} + +VALUE +rb_gc_impl_stress_get(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + return ruby_gc_stress_mode; +} + +void +rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag) +{ + rb_objspace_t *objspace = objspace_ptr; + + objspace->flags.gc_stressful = RTEST(flag); + objspace->gc_stress_mode = flag; +} + +static int +get_envparam_size(const char *name, size_t *default_value, size_t lower_bound) +{ + const char *ptr = getenv(name); + ssize_t val; + + if (ptr != NULL && *ptr) { + size_t unit = 0; + char *end; +#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG + val = strtoll(ptr, &end, 0); +#else + val = strtol(ptr, &end, 0); +#endif + switch (*end) { + case 'k': case 'K': + unit = 1024; + ++end; + break; + case 'm': case 'M': + unit = 1024*1024; + ++end; + break; + case 'g': case 'G': + unit = 1024*1024*1024; + ++end; + break; + } + while (*end && isspace((unsigned char)*end)) end++; + if (*end) { + if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr); + return 0; + } + if (unit > 0) { + if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) { + if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr); + return 0; + } + val *= unit; + } + if (val > 0 && (size_t)val > lower_bound) { + if (RTEST(ruby_verbose)) { + fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value); + } + *default_value = (size_t)val; + return 1; + } + else { + if (RTEST(ruby_verbose)) { + fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n", + name, val, *default_value, lower_bound); + } + return 0; + } + } + return 0; +} + +static int +get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero) +{ + const char *ptr = getenv(name); + double val; + + if (ptr != NULL && *ptr) { + char *end; + val = strtod(ptr, &end); + if (!*ptr || *end) { + if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr); + return 0; + } + + if (accept_zero && val == 0.0) { + goto accept; + } + else if (val <= lower_bound) { + if (RTEST(ruby_verbose)) { + fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n", + name, val, *default_value, lower_bound); + } + } + else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */ + val > upper_bound) { + if (RTEST(ruby_verbose)) { + fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n", + name, val, *default_value, upper_bound); + } + } + else { + goto accept; + } + } + return 0; + + accept: + if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value); + *default_value = val; + return 1; +} + +/* + * GC tuning environment variables + * + * * RUBY_GC_HEAP_FREE_SLOTS + * - Prepare at least this amount of slots after GC. + * - Allocate slots if there are not enough slots. + * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1) + * - Allocate slots by this factor. + * - (next slots number) = (current slots number) * (this factor) + * * RUBY_GC_HEAP_GROWTH_MAX_BYTES (was RUBY_GC_HEAP_GROWTH_MAX_SLOTS) + * - Allocation rate is limited to this number of bytes. + * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4) + * - Allocate additional pages when the number of free slots is + * lower than the value (total_slots * (this ratio)). + * * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4) + * - Allocate slots to satisfy this formula: + * free_slots = total_slots * goal_ratio + * - In other words, prepare (total_slots * goal_ratio) free slots. + * - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly. + * * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4) + * - Allow to free pages when the number of free slots is + * greater than the value (total_slots * (this ratio)). + * * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1) + * - Do full GC when the number of old objects is more than R * N + * where R is this factor and + * N is the number of old objects just after last full GC. + * + * * obsolete + * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1) + * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) -> RUBY_GC_HEAP_INIT_BYTES + * + * * RUBY_GC_MALLOC_LIMIT + * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1) + * * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1) + * + * * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1) + * * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1) + * * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1) + */ + +void +rb_gc_impl_set_params(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + /* RUBY_GC_HEAP_FREE_SLOTS */ + if (get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) { + /* ok */ + } + + get_envparam_size("RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes, 0); + + get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE); + get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0); + get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio, + 0.0, 1.0, FALSE); + get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio, + gc_params.heap_free_slots_min_ratio, 1.0, FALSE); + get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio, + gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE); + get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE); + get_envparam_double("RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE); + + if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) { + malloc_limit = gc_params.malloc_limit_min; + } + get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0); + if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */ + gc_params.malloc_limit_max = SIZE_MAX; + } + get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE); + +#if RGENGC_ESTIMATE_OLDMALLOC + if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) { + objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min; + } + get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0); + get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE); +#endif +} + +static inline size_t +objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint) +{ +#ifdef HAVE_MALLOC_USABLE_SIZE + if (!hint) { + hint = malloc_usable_size(ptr); + } +#endif + return hint; +} + +enum memop_type { + MEMOP_TYPE_MALLOC = 0, + MEMOP_TYPE_FREE, + MEMOP_TYPE_REALLOC +}; + +static inline void +atomic_sub_nounderflow(size_t *var, size_t sub) +{ + if (sub == 0) return; + + while (1) { + size_t val = *var; + if (val < sub) sub = val; + if (RUBY_ATOMIC_SIZE_CAS(*var, val, val-sub) == val) break; + } +} + +#define gc_stress_full_mark_after_malloc_p() \ + (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc))) + +static void +objspace_malloc_gc_stress(rb_objspace_t *objspace) +{ + if (ruby_gc_stressful && ruby_native_thread_p()) { + unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP | + GPR_FLAG_STRESS | GPR_FLAG_MALLOC); + + if (gc_stress_full_mark_after_malloc_p()) { + reason |= GPR_FLAG_FULL_MARK; + } + garbage_collect_with_gvl(objspace, reason); + } +} + +static void +malloc_increase_commit(rb_objspace_t *objspace, size_t new_size, size_t old_size) +{ + if (new_size > old_size) { + size_t delta = new_size - old_size; + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_add(&objspace->malloc_counters.counters.malloc, delta); +#if RGENGC_ESTIMATE_OLDMALLOC + gc_counter_add(&objspace->malloc_counters.oldcounters.malloc, delta); +#endif + MALLOC_COUNTERS_UNLOCK(objspace); + } + else if (old_size > new_size) { + size_t delta = old_size - new_size; + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_add(&objspace->malloc_counters.counters.free, delta); +#if RGENGC_ESTIMATE_OLDMALLOC + gc_counter_add(&objspace->malloc_counters.oldcounters.free, delta); +#endif + MALLOC_COUNTERS_UNLOCK(objspace); + } +} + +#if USE_MALLOC_INCREASE_LOCAL +static void +malloc_increase_local_flush(rb_objspace_t *objspace) +{ + int delta = malloc_increase_local; + if (delta == 0) return; + + malloc_increase_local = 0; + if (delta > 0) { + malloc_increase_commit(objspace, (size_t)delta, 0); + } + else { + malloc_increase_commit(objspace, 0, (size_t)(-delta)); + } +} +#else +static void +malloc_increase_local_flush(rb_objspace_t *objspace) +{ +} +#endif + +static inline bool +objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed) +{ + if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n", + mem, + type == MEMOP_TYPE_MALLOC ? "malloc" : + type == MEMOP_TYPE_FREE ? "free " : + type == MEMOP_TYPE_REALLOC ? "realloc": "error", + new_size, old_size); + return false; +} + +static bool +objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed) +{ +#if USE_MALLOC_INCREASE_LOCAL + if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD && + old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) { + malloc_increase_local += (int)new_size - (int)old_size; + + if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD || + malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) { + malloc_increase_local_flush(objspace); + } + } + else { + malloc_increase_local_flush(objspace); + malloc_increase_commit(objspace, new_size, old_size); + } +#else + malloc_increase_commit(objspace, new_size, old_size); +#endif + + if (type == MEMOP_TYPE_MALLOC && gc_allowed) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { + if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) { + gc_rest(objspace); /* gc_rest can reduce malloc_increase */ + goto retry; + } + garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC); + } + } + +#if MALLOC_ALLOCATED_SIZE + if (new_size >= old_size) { + RUBY_ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size); + } + else { + size_t dec_size = old_size - new_size; + +#if MALLOC_ALLOCATED_SIZE_CHECK + size_t allocated_size = objspace->malloc_params.allocated_size; + if (allocated_size < dec_size) { + rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size."); + } +#endif + atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size); + } + + switch (type) { + case MEMOP_TYPE_MALLOC: + RUBY_ATOMIC_SIZE_INC(objspace->malloc_params.allocations); + break; + case MEMOP_TYPE_FREE: + { + size_t allocations = objspace->malloc_params.allocations; + if (allocations > 0) { + atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1); + } +#if MALLOC_ALLOCATED_SIZE_CHECK + else { + GC_ASSERT(objspace->malloc_params.allocations > 0); + } +#endif + } + break; + case MEMOP_TYPE_REALLOC: /* ignore */ break; + } +#endif + return true; +} + +#define objspace_malloc_increase(...) \ + for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \ + !malloc_increase_done; \ + malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__)) + +struct malloc_obj_info { /* 4 words */ + size_t size; +}; + +static inline size_t +objspace_malloc_prepare(rb_objspace_t *objspace, size_t size) +{ + if (size == 0) size = 1; + +#if CALC_EXACT_MALLOC_SIZE + size += sizeof(struct malloc_obj_info); +#endif + + return size; +} + +static bool +malloc_during_gc_p(rb_objspace_t *objspace) +{ + /* malloc is not allowed during GC when we're not using multiple ractors + * (since ractors can run while another thread is sweeping) and when we + * have the GVL (since if we don't have the GVL, we'll try to acquire the + * GVL which will block and ensure the other thread finishes GC). */ + return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p(); +} + +static inline void * +objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size, bool gc_allowed) +{ + size = objspace_malloc_size(objspace, mem, size); + objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {} + +#if CALC_EXACT_MALLOC_SIZE + { + struct malloc_obj_info *info = mem; + info->size = size; + mem = info + 1; + } +#endif + + return mem; +} + +#if defined(__GNUC__) && RUBY_DEBUG +#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1 +#endif + +#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR +# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0 +#endif + +#define GC_MEMERROR(...) \ + ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0) + +#define TRY_WITH_GC(siz, expr) do { \ + const gc_profile_record_flag gpr = \ + GPR_FLAG_FULL_MARK | \ + GPR_FLAG_IMMEDIATE_MARK | \ + GPR_FLAG_IMMEDIATE_SWEEP | \ + GPR_FLAG_MALLOC; \ + objspace_malloc_gc_stress(objspace); \ + \ + if (RB_LIKELY((expr))) { \ + /* Success on 1st try */ \ + } \ + else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \ + /* @shyouhei thinks this doesn't happen */ \ + GC_MEMERROR("TRY_WITH_GC: could not GC"); \ + } \ + else if ((expr)) { \ + /* Success on 2nd try */ \ + } \ + else { \ + GC_MEMERROR("TRY_WITH_GC: could not allocate:" \ + "%"PRIdSIZE" bytes for %s", \ + siz, # expr); \ + } \ + } while (0) + +static void +check_malloc_not_in_gc(rb_objspace_t *objspace, const char *msg) +{ + if (RB_UNLIKELY(malloc_during_gc_p(objspace))) { + dont_gc_on(); + during_gc = false; + rb_bug("Cannot %s during GC", msg); + } +} + +void +rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (!ptr) { + /* + * ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since + * its first version. We would better follow. + */ + return; + } +#if CALC_EXACT_MALLOC_SIZE + struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; +#if VERIFY_FREE_SIZE + if (!info->size) { + rb_bug("buffer %p has no recorded size. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr); + } + + if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) { + rb_bug("buffer %p freed with old_size=%zu, but was allocated with size=%zu", ptr, old_size, info->size - sizeof(struct malloc_obj_info)); + } +#endif + ptr = info; + old_size = info->size; +#endif + old_size = objspace_malloc_size(objspace, ptr, old_size); + + objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE, true) { + free(ptr); + ptr = NULL; + RB_DEBUG_COUNTER_INC(heap_xfree); + } +} + +void * +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + rb_objspace_t *objspace = objspace_ptr; + check_malloc_not_in_gc(objspace, "malloc"); + + void *mem; + + size = objspace_malloc_prepare(objspace, size); + TRY_WITH_GC(size, mem = malloc(size)); + RB_DEBUG_COUNTER_INC(heap_xmalloc); + if (!mem) return mem; + return objspace_malloc_fixup(objspace, mem, size, gc_allowed); +} + +void * +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (RB_UNLIKELY(malloc_during_gc_p(objspace))) { + rb_warn("calloc during GC detected, this could cause crashes if it triggers another GC"); +#if RGENGC_CHECK_MODE || RUBY_DEBUG + rb_bug("Cannot calloc during GC"); +#endif + } + + void *mem; + + size = objspace_malloc_prepare(objspace, size); + TRY_WITH_GC(size, mem = calloc1(size)); + if (!mem) return mem; + return objspace_malloc_fixup(objspace, mem, size, gc_allowed); +} + +void * +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) +{ + rb_objspace_t *objspace = objspace_ptr; + + check_malloc_not_in_gc(objspace, "realloc"); + + void *mem; + + if (!ptr) return rb_gc_impl_malloc(objspace, new_size, gc_allowed); + + /* + * The behavior of realloc(ptr, 0) is implementation defined. + * Therefore we don't use realloc(ptr, 0) for portability reason. + * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm + */ + if (new_size == 0) { + if ((mem = rb_gc_impl_malloc(objspace, 0, gc_allowed)) != NULL) { + /* + * - OpenBSD's malloc(3) man page says that when 0 is passed, it + * returns a non-NULL pointer to an access-protected memory page. + * The returned pointer cannot be read / written at all, but + * still be a valid argument of free(). + * + * https://man.openbsd.org/malloc.3 + * + * - Linux's malloc(3) man page says that it _might_ perhaps return + * a non-NULL pointer when its argument is 0. That return value + * is safe (and is expected) to be passed to free(). + * + * https://man7.org/linux/man-pages/man3/malloc.3.html + * + * - As I read the implementation jemalloc's malloc() returns fully + * normal 16 bytes memory region when its argument is 0. + * + * - As I read the implementation musl libc's malloc() returns + * fully normal 32 bytes memory region when its argument is 0. + * + * - Other malloc implementations can also return non-NULL. + */ + rb_gc_impl_free(objspace, ptr, old_size); + return mem; + } + else { + /* + * It is dangerous to return NULL here, because that could lead to + * RCE. Fallback to 1 byte instead of zero. + * + * https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932 + */ + new_size = 1; + } + } + +#if CALC_EXACT_MALLOC_SIZE + { + struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; + new_size += sizeof(struct malloc_obj_info); + ptr = info; +#if VERIFY_FREE_SIZE + if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) { + rb_bug("buffer %p realloced with old_size=%zu, but was allocated with size=%zu", ptr, old_size, info->size - sizeof(struct malloc_obj_info)); + } +#endif + old_size = info->size; + } +#endif + + old_size = objspace_malloc_size(objspace, ptr, old_size); + TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size))); + if (!mem) return mem; + new_size = objspace_malloc_size(objspace, mem, new_size); + +#if CALC_EXACT_MALLOC_SIZE + { + struct malloc_obj_info *info = mem; + info->size = new_size; + mem = info + 1; + } +#endif + + objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed); + + RB_DEBUG_COUNTER_INC(heap_xrealloc); + return mem; +} + +void +rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (diff > 0) { + objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC, true); + } + else if (diff < 0) { + objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC, true); + } +} + +// TODO: move GC profiler stuff back into gc.c +/* + ------------------------------ GC profiler ------------------------------ +*/ + +#define GC_PROFILE_RECORD_DEFAULT_SIZE 100 + +static bool +current_process_time(struct timespec *ts) +{ +#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) + { + static int try_clock_gettime = 1; + if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) { + return true; + } + else { + try_clock_gettime = 0; + } + } +#endif + +#ifdef RUSAGE_SELF + { + struct rusage usage; + struct timeval time; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + time = usage.ru_utime; + ts->tv_sec = time.tv_sec; + ts->tv_nsec = (int32_t)time.tv_usec * 1000; + return true; + } + } +#endif + +#ifdef _WIN32 + { + FILETIME creation_time, exit_time, kernel_time, user_time; + ULARGE_INTEGER ui; + + if (GetProcessTimes(GetCurrentProcess(), + &creation_time, &exit_time, &kernel_time, &user_time) != 0) { + memcpy(&ui, &user_time, sizeof(FILETIME)); +#define PER100NSEC (uint64_t)(1000 * 1000 * 10) + ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC); + ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC); + return true; + } + } +#endif + + return false; +} + +static double +getrusage_time(void) +{ + struct timespec ts; + if (current_process_time(&ts)) { + return ts.tv_sec + ts.tv_nsec * 1e-9; + } + else { + return 0.0; + } +} + + +static inline void +gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason) +{ + if (objspace->profile.run) { + size_t index = objspace->profile.next_index; + gc_profile_record *record; + + /* create new record */ + objspace->profile.next_index++; + + if (!objspace->profile.records) { + objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE; + objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size)); + } + if (index >= objspace->profile.size) { + void *ptr; + objspace->profile.size += 1000; + ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size)); + if (!ptr) rb_memerror(); + objspace->profile.records = ptr; + } + if (!objspace->profile.records) { + rb_bug("gc_profile malloc or realloc miss"); + } + record = objspace->profile.current_record = &objspace->profile.records[objspace->profile.next_index - 1]; + MEMZERO(record, gc_profile_record, 1); + + /* setup before-GC parameter */ + record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0); +#if MALLOC_ALLOCATED_SIZE + record->allocated_size = malloc_allocated_size; +#endif +#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY +#ifdef RUSAGE_SELF + { + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + record->maxrss = usage.ru_maxrss; + record->minflt = usage.ru_minflt; + record->majflt = usage.ru_majflt; + } + } +#endif +#endif + } +} + +static inline void +gc_prof_timer_start(rb_objspace_t *objspace) +{ + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); +#if GC_PROFILE_MORE_DETAIL + record->prepare_time = objspace->profile.prepare_time; +#endif + record->gc_time = 0; + record->gc_invoke_time = getrusage_time(); + } +} + +static double +elapsed_time_from(double time) +{ + double now = getrusage_time(); + if (now > time) { + return now - time; + } + else { + return 0; + } +} + +static inline void +gc_prof_timer_stop(rb_objspace_t *objspace) +{ + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->gc_time = elapsed_time_from(record->gc_invoke_time); + record->gc_invoke_time -= objspace->profile.invoke_time; + } +} + +#ifdef BUILDING_MODULAR_GC +# define RUBY_DTRACE_GC_HOOK(name) +#else +# define RUBY_DTRACE_GC_HOOK(name) \ + do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0) +#endif + +static inline void +gc_prof_mark_timer_start(rb_objspace_t *objspace) +{ + RUBY_DTRACE_GC_HOOK(MARK_BEGIN); +#if GC_PROFILE_MORE_DETAIL + if (gc_prof_enabled(objspace)) { + gc_prof_record(objspace)->gc_mark_time = getrusage_time(); + } +#endif +} + +static inline void +gc_prof_mark_timer_stop(rb_objspace_t *objspace) +{ + RUBY_DTRACE_GC_HOOK(MARK_END); +#if GC_PROFILE_MORE_DETAIL + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->gc_mark_time = elapsed_time_from(record->gc_mark_time); + } +#endif +} + +static inline void +gc_prof_sweep_timer_start(rb_objspace_t *objspace) +{ + RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN); + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + + if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) { + objspace->profile.gc_sweep_start_time = getrusage_time(); + } + } +} + +static inline void +gc_prof_sweep_timer_stop(rb_objspace_t *objspace) +{ + RUBY_DTRACE_GC_HOOK(SWEEP_END); + + if (gc_prof_enabled(objspace)) { + double sweep_time; + gc_profile_record *record = gc_prof_record(objspace); + + if (record->gc_time > 0) { + sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time); + /* need to accumulate GC time for lazy sweep after gc() */ + record->gc_time += sweep_time; + } + else if (GC_PROFILE_MORE_DETAIL) { + sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time); + } + +#if GC_PROFILE_MORE_DETAIL + record->gc_sweep_time += sweep_time; + if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE; +#endif + if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE; + } +} + +static inline void +gc_prof_set_malloc_info(rb_objspace_t *objspace) +{ +#if GC_PROFILE_MORE_DETAIL + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + record->allocate_increase = malloc_increase; + record->allocate_limit = malloc_limit; + } +#endif +} + +static inline void +gc_prof_set_heap_info(rb_objspace_t *objspace) +{ + if (gc_prof_enabled(objspace)) { + gc_profile_record *record = gc_prof_record(objspace); + + /* Sum across all size pools since each has a different slot size. */ + size_t total = 0; + size_t use_size = 0; + size_t total_size = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count; + total += heap->total_slots; + use_size += heap_live * heap->slot_size; + total_size += heap->total_slots * heap->slot_size; + } + +#if GC_PROFILE_MORE_DETAIL + size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace); + record->heap_use_pages = objspace->profile.heap_used_at_gc_start; + record->heap_live_objects = live; + record->heap_free_objects = total - live; +#endif + + record->heap_total_objects = total; + record->heap_use_size = use_size; + record->heap_total_size = total_size; + } +} + +/* + * call-seq: + * GC::Profiler.clear -> nil + * + * Clears the \GC profiler data. + * + */ + +static VALUE +gc_profile_clear(VALUE _) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + void *p = objspace->profile.records; + objspace->profile.records = NULL; + objspace->profile.size = 0; + objspace->profile.next_index = 0; + objspace->profile.current_record = 0; + free(p); + return Qnil; +} + +/* + * call-seq: + * GC::Profiler.raw_data -> [Hash, ...] + * + * Returns an Array of individual raw profile data Hashes ordered + * from earliest to latest by +:GC_INVOKE_TIME+. + * + * For example: + * + * [ + * { + * :GC_TIME=>1.3000000000000858e-05, + * :GC_INVOKE_TIME=>0.010634999999999999, + * :HEAP_USE_SIZE=>289640, + * :HEAP_TOTAL_SIZE=>588960, + * :HEAP_TOTAL_OBJECTS=>14724, + * :GC_IS_MARKED=>false + * }, + * # ... + * ] + * + * The keys mean: + * + * +:GC_TIME+:: + * Time elapsed in seconds for this GC run + * +:GC_INVOKE_TIME+:: + * Time elapsed in seconds from startup to when the GC was invoked + * +:HEAP_USE_SIZE+:: + * Total bytes of heap used + * +:HEAP_TOTAL_SIZE+:: + * Total size of heap in bytes + * +:HEAP_TOTAL_OBJECTS+:: + * Total number of objects + * +:GC_IS_MARKED+:: + * Returns +true+ if the GC is in mark phase + * + * If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access + * to the following hash keys: + * + * +:GC_MARK_TIME+:: + * +:GC_SWEEP_TIME+:: + * +:ALLOCATE_INCREASE+:: + * +:ALLOCATE_LIMIT+:: + * +:HEAP_USE_PAGES+:: + * +:HEAP_LIVE_OBJECTS+:: + * +:HEAP_FREE_OBJECTS+:: + * +:HAVE_FINALIZE+:: + * + */ + +static VALUE +gc_profile_record_get(VALUE _) +{ + VALUE prof; + VALUE gc_profile = rb_ary_new(); + size_t i; + rb_objspace_t *objspace = rb_gc_get_objspace(); + + if (!objspace->profile.run) { + return Qnil; + } + + for (i =0; i < objspace->profile.next_index; i++) { + gc_profile_record *record = &objspace->profile.records[i]; + + prof = rb_hash_new(); + rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(objspace, rb_hash_new(), record->flags)); + rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time)); + rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time)); + rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size)); + rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size)); + rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects)); + rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects)); + rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue); +#if GC_PROFILE_MORE_DETAIL + rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time)); + rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time)); + rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase)); + rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit)); + rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages)); + rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects)); + rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects)); + + rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects)); + rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects)); + + rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse); +#endif + +#if RGENGC_PROFILE > 0 + rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects)); + rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects)); + rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects)); +#endif + rb_ary_push(gc_profile, prof); + } + + return gc_profile; +} + +#if GC_PROFILE_MORE_DETAIL +#define MAJOR_REASON_MAX 0x10 + +static char * +gc_profile_dump_major_reason(unsigned int flags, char *buff) +{ + unsigned int reason = flags & GPR_FLAG_MAJOR_MASK; + int i = 0; + + if (reason == GPR_FLAG_NONE) { + buff[0] = '-'; + buff[1] = 0; + } + else { +#define C(x, s) \ + if (reason & GPR_FLAG_MAJOR_BY_##x) { \ + buff[i++] = #x[0]; \ + if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \ + buff[i] = 0; \ + } + C(NOFREE, N); + C(OLDGEN, O); + C(SHADY, S); +#if RGENGC_ESTIMATE_OLDMALLOC + C(OLDMALLOC, M); +#endif +#undef C + } + return buff; +} +#endif + + + +static void +gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE)) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + size_t count = objspace->profile.next_index; +#ifdef MAJOR_REASON_MAX + char reason_str[MAJOR_REASON_MAX]; +#endif + + if (objspace->profile.run && count /* > 1 */) { + size_t i; + const gc_profile_record *record; + + append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count)); + append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n")); + + for (i = 0; i < count; i++) { + record = &objspace->profile.records[i]; + append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n", + i+1, record->gc_invoke_time, record->heap_use_size, + record->heap_total_size, record->heap_total_objects, record->gc_time*1000)); + } + +#if GC_PROFILE_MORE_DETAIL + const char *str = "\n\n" \ + "More detail.\n" \ + "Prepare Time = Previously GC's rest sweep time\n" + "Index Flags Allocate Inc. Allocate Limit" +#if CALC_EXACT_MALLOC_SIZE + " Allocated Size" +#endif + " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj" +#if RGENGC_PROFILE + " OldgenObj RemNormObj RemShadObj" +#endif +#if GC_PROFILE_DETAIL_MEMORY + " MaxRSS(KB) MinorFLT MajorFLT" +#endif + "\n"; + append(out, rb_str_new_cstr(str)); + + for (i = 0; i < count; i++) { + record = &objspace->profile.records[i]; + append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE +#if CALC_EXACT_MALLOC_SIZE + " %15"PRIuSIZE +#endif + " %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE +#if RGENGC_PROFILE + "%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE +#endif +#if GC_PROFILE_DETAIL_MEMORY + "%11ld %8ld %8ld" +#endif + + "\n", + i+1, + gc_profile_dump_major_reason(record->flags, reason_str), + (record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.', + (record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" : + (record->flags & GPR_FLAG_MALLOC) ? "MALLOC" : + (record->flags & GPR_FLAG_METHOD) ? "METHOD" : + (record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????", + (record->flags & GPR_FLAG_STRESS) ? '!' : ' ', + record->allocate_increase, record->allocate_limit, +#if CALC_EXACT_MALLOC_SIZE + record->allocated_size, +#endif + record->heap_use_pages, + record->gc_mark_time*1000, + record->gc_sweep_time*1000, + record->prepare_time*1000, + + record->heap_live_objects, + record->heap_free_objects, + record->removing_objects, + record->empty_objects +#if RGENGC_PROFILE + , + record->old_objects, + record->remembered_normal_objects, + record->remembered_shady_objects +#endif +#if GC_PROFILE_DETAIL_MEMORY + , + record->maxrss / 1024, + record->minflt, + record->majflt +#endif + + )); + } +#endif + } +} + +/* + * call-seq: + * GC::Profiler.result -> String + * + * Returns a profile data report such as: + * + * GC 1 invokes. + * Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms) + * 1 0.012 159240 212940 10647 0.00000000000001530000 + */ + +static VALUE +gc_profile_result(VALUE _) +{ + VALUE str = rb_str_buf_new(0); + gc_profile_dump_on(str, rb_str_buf_append); + return str; +} + +/* + * call-seq: + * GC::Profiler.report + * GC::Profiler.report(io) + * + * Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object. + * + */ + +static VALUE +gc_profile_report(int argc, VALUE *argv, VALUE self) +{ + VALUE out; + + out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]); + gc_profile_dump_on(out, rb_io_write); + + return Qnil; +} + +/* + * call-seq: + * GC::Profiler.total_time -> float + * + * The total time used for garbage collection in seconds + */ + +static VALUE +gc_profile_total_time(VALUE self) +{ + double time = 0; + rb_objspace_t *objspace = rb_gc_get_objspace(); + + if (objspace->profile.run && objspace->profile.next_index > 0) { + size_t i; + size_t count = objspace->profile.next_index; + + for (i = 0; i < count; i++) { + time += objspace->profile.records[i].gc_time; + } + } + return DBL2NUM(time); +} + +/* + * call-seq: + * GC::Profiler.enabled? -> true or false + * + * The current status of \GC profile mode. + */ + +static VALUE +gc_profile_enable_get(VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + return objspace->profile.run ? Qtrue : Qfalse; +} + +/* + * call-seq: + * GC::Profiler.enable -> nil + * + * Starts the \GC profiler. + * + */ + +static VALUE +gc_profile_enable(VALUE _) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + objspace->profile.run = TRUE; + objspace->profile.current_record = 0; + return Qnil; +} + +/* + * call-seq: + * GC::Profiler.disable -> nil + * + * Stops the \GC profiler. + * + */ + +static VALUE +gc_profile_disable(VALUE _) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + + objspace->profile.run = FALSE; + objspace->profile.current_record = 0; + return Qnil; +} + +void +rb_gc_verify_internal_consistency(void) +{ + gc_verify_internal_consistency(rb_gc_get_objspace()); +} + +/* + * call-seq: + * GC.verify_internal_consistency -> nil + * + * Verify internal consistency. + * + * This method is implementation specific. + * Now this method checks generational consistency + * if RGenGC is supported. + */ +static VALUE +gc_verify_internal_consistency_m(VALUE dummy) +{ + rb_gc_verify_internal_consistency(); + return Qnil; +} + +#if GC_CAN_COMPILE_COMPACTION +/* + * call-seq: + * GC.auto_compact = flag + * + * Updates automatic compaction mode. + * + * When enabled, the compactor will execute on every major collection. + * + * Enabling compaction will degrade performance on major collections. + */ +static VALUE +gc_set_auto_compact(VALUE _, VALUE v) +{ + GC_ASSERT(GC_COMPACTION_SUPPORTED); + + ruby_enable_autocompact = RTEST(v); + +#if RGENGC_CHECK_MODE + ruby_autocompact_compare_func = NULL; + + if (SYMBOL_P(v)) { + ID id = RB_SYM2ID(v); + if (id == rb_intern("empty")) { + ruby_autocompact_compare_func = compare_free_slots; + } + } +#endif + + return v; +} +#else +# define gc_set_auto_compact rb_f_notimplement +#endif + +#if GC_CAN_COMPILE_COMPACTION +/* + * call-seq: + * GC.auto_compact -> true or false + * + * Returns whether or not automatic compaction has been enabled. + */ +static VALUE +gc_get_auto_compact(VALUE _) +{ + return ruby_enable_autocompact ? Qtrue : Qfalse; +} +#else +# define gc_get_auto_compact rb_f_notimplement +#endif + +#if GC_CAN_COMPILE_COMPACTION +/* + * call-seq: + * GC.latest_compact_info -> hash + * + * Returns information about object moved in the most recent \GC compaction. + * + * The returned +hash+ contains the following keys: + * + * [considered] + * Hash containing the type of the object as the key and the number of + * objects of that type that were considered for movement. + * [moved] + * Hash containing the type of the object as the key and the number of + * objects of that type that were actually moved. + * [moved_up] + * Hash containing the type of the object as the key and the number of + * objects of that type that were increased in size. + * [moved_down] + * Hash containing the type of the object as the key and the number of + * objects of that type that were decreased in size. + * + * Some objects can't be moved (due to pinning) so these numbers can be used to + * calculate compaction efficiency. + */ +static VALUE +gc_compact_stats(VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + VALUE h = rb_hash_new(); + VALUE considered = rb_hash_new(); + VALUE moved = rb_hash_new(); + VALUE moved_up = rb_hash_new(); + VALUE moved_down = rb_hash_new(); + + for (size_t i = 0; i < T_MASK; i++) { + if (objspace->rcompactor.considered_count_table[i]) { + rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i])); + } + + if (objspace->rcompactor.moved_count_table[i]) { + rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i])); + } + + if (objspace->rcompactor.moved_up_count_table[i]) { + rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i])); + } + + if (objspace->rcompactor.moved_down_count_table[i]) { + rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i])); + } + } + + rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered); + rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved); + rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up); + rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down); + + return h; +} +#else +# define gc_compact_stats rb_f_notimplement +#endif + +#if GC_CAN_COMPILE_COMPACTION +/* + * call-seq: + * GC.compact -> hash + * + * This function compacts objects together in Ruby's heap. It eliminates + * unused space (or fragmentation) in the heap by moving objects in to that + * unused space. + * + * The returned +hash+ contains statistics about the objects that were moved; + * see GC.latest_compact_info. + * + * This method is only expected to work on CRuby. + * + * To test whether \GC compaction is supported, use the idiom: + * + * GC.respond_to?(:compact) + */ +static VALUE +gc_compact(VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + int full_marking_p = gc_config_full_mark_val; + gc_config_full_mark_set(TRUE); + + /* Run GC with compaction enabled */ + rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true); + gc_config_full_mark_set(full_marking_p); + + return gc_compact_stats(self); +} +#else +# define gc_compact rb_f_notimplement +#endif + +#if GC_CAN_COMPILE_COMPACTION +struct desired_compaction_pages_i_data { + rb_objspace_t *objspace; + size_t required_slots[HEAP_COUNT]; +}; + +static int +desired_compaction_pages_i(struct heap_page *page, void *data) +{ + struct desired_compaction_pages_i_data *tdata = data; + rb_objspace_t *objspace = tdata->objspace; + VALUE vstart = (VALUE)page->start; + VALUE vend = vstart + (VALUE)(page->total_slots * page->heap->slot_size); + + + for (VALUE v = vstart; v != vend; v += page->heap->slot_size) { + asan_unpoisoning_object(v) { + /* skip T_NONEs; they won't be moved */ + if (BUILTIN_TYPE(v) != T_NONE) { + rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, page->heap, v); + size_t dest_pool_idx = dest_pool - heaps; + tdata->required_slots[dest_pool_idx]++; + } + } + } + + return 0; +} + +/* call-seq: + * GC.verify_compaction_references(toward: nil, double_heap: false) -> hash + * + * Verify compaction reference consistency. + * + * This method is implementation specific. During compaction, objects that + * were moved are replaced with T_MOVED objects. No object should have a + * reference to a T_MOVED object after compaction. + * + * This function expands the heap to ensure room to move all objects, + * compacts the heap to make sure everything moves, updates all references, + * then performs a full \GC. If any object contains a reference to a T_MOVED + * object, that object should be pushed on the mark stack, and will + * make a SEGV. + */ +static VALUE +gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) +{ + static ID keywords[3] = {0}; + if (!keywords[0]) { + keywords[0] = rb_intern("toward"); + keywords[1] = rb_intern("double_heap"); + keywords[2] = rb_intern("expand_heap"); + } + + VALUE options; + rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options); + + VALUE arguments[3] = { Qnil, Qfalse, Qfalse }; + int kwarg_count = rb_get_kwargs(options, keywords, 0, 3, arguments); + bool toward_empty = kwarg_count > 0 && SYMBOL_P(arguments[0]) && SYM2ID(arguments[0]) == rb_intern("empty"); + bool expand_heap = (kwarg_count > 1 && RTEST(arguments[1])) || (kwarg_count > 2 && RTEST(arguments[2])); + + rb_objspace_t *objspace = rb_gc_get_objspace(); + + /* Clear the heap. */ + rb_gc_impl_start(objspace, true, true, true, false); + + unsigned int lev = RB_GC_VM_LOCK(); + { + gc_rest(objspace); + + /* if both double_heap and expand_heap are set, expand_heap takes precedence */ + if (expand_heap) { + struct desired_compaction_pages_i_data desired_compaction = { + .objspace = objspace, + .required_slots = {0}, + }; + /* Work out how many objects want to be in each size pool, taking account of moves */ + objspace_each_pages(objspace, desired_compaction_pages_i, &desired_compaction, TRUE); + + /* Find out which pool has the most pages */ + size_t max_existing_pages = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + max_existing_pages = MAX(max_existing_pages, heap->total_pages); + } + + /* Add pages to each size pool so that compaction is guaranteed to move every object */ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + size_t pages_to_add = 0; + /* + * Step 1: Make sure every pool has the same number of pages, by adding empty pages + * to smaller pools. This is required to make sure the compact cursor can advance + * through all of the pools in `gc_sweep_compact` without hitting the "sweep & + * compact cursors met" condition on some pools before fully compacting others + */ + pages_to_add += max_existing_pages - heap->total_pages; + /* + * Step 2: Now add additional free pages to each size pool sufficient to hold all objects + * that want to be in that size pool, whether moved into it or moved within it + */ + objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size; + while (objspace->heap_pages.allocatable_bytes > 0) { + heap_page_allocate_and_initialize(objspace, heap); + } + /* + * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects + * have been moved, and not on the last iteration of the `gc_sweep_compact` loop + */ + pages_to_add += 2; + + for (; pages_to_add > 0; pages_to_add--) { + heap_page_allocate_and_initialize_force(objspace, heap); + } + } + } + + if (toward_empty) { + objspace->rcompactor.compare_func = compare_free_slots; + } + } + RB_GC_VM_UNLOCK(lev); + + rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true); + + rb_objspace_reachable_objects_from_root(root_obj_check_moved_i, objspace); + objspace_each_objects(objspace, heap_check_moved_i, objspace, TRUE); + + objspace->rcompactor.compare_func = NULL; + + return gc_compact_stats(self); +} +#else +# define gc_verify_compaction_references rb_f_notimplement +#endif + +void +rb_gc_impl_objspace_free(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + if (is_lazy_sweeping(objspace)) + rb_bug("lazy sweeping underway when freeing object space"); + + free(objspace->profile.records); + objspace->profile.records = NULL; + + for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { + heap_page_free(objspace, rb_darray_get(objspace->heap_pages.sorted, i)); + } + rb_darray_free_without_gc(objspace->heap_pages.sorted); + heap_pages_lomem = 0; + heap_pages_himem = 0; + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + heap->total_pages = 0; + heap->total_slots = 0; + } + + free_stack_chunks(&objspace->mark_stack); + mark_stack_free_cache(&objspace->mark_stack); + + rb_darray_free_without_gc(objspace->weak_references); + +#ifdef MALLOC_COUNTERS_NEED_LOCK + rb_native_mutex_destroy(&objspace->malloc_counters.lock); +#endif + + free(objspace); +} + +#if MALLOC_ALLOCATED_SIZE +/* + * call-seq: + * GC.malloc_allocated_size -> Integer + * + * Returns the size of memory allocated by malloc(). + * + * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+. + */ + +static VALUE +gc_malloc_allocated_size(VALUE self) +{ + rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace(); + return ULL2NUM(objspace->malloc_params.allocated_size); +} + +/* + * call-seq: + * GC.malloc_allocations -> Integer + * + * Returns the number of malloc() allocations. + * + * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+. + */ + +static VALUE +gc_malloc_allocations(VALUE self) +{ + rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace(); + return ULL2NUM(objspace->malloc_params.allocations); +} +#endif + +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + objspace->fork_vm_lock_lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + rb_objspace_t *objspace = objspace_ptr; + + RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev); + objspace->fork_vm_lock_lev = 0; + + if (pid == 0) { /* child process */ + rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL); + } +} + +VALUE rb_ident_hash_new_with_size(st_index_t size); + +#if GC_DEBUG_STRESS_TO_CLASS +/* + * call-seq: + * GC.add_stress_to_class(class[, ...]) + * + * Raises NoMemoryError when allocating an instance of the given classes. + * + */ +static VALUE +rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + + if (!stress_to_class) { + set_stress_to_class(rb_ident_hash_new_with_size(argc)); + } + + for (int i = 0; i < argc; i++) { + VALUE klass = argv[i]; + rb_hash_aset(stress_to_class, klass, Qtrue); + } + + return self; +} + +/* + * call-seq: + * GC.remove_stress_to_class(class[, ...]) + * + * No longer raises NoMemoryError when allocating an instance of the + * given classes. + * + */ +static VALUE +rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + + if (stress_to_class) { + for (int i = 0; i < argc; ++i) { + rb_hash_delete(stress_to_class, argv[i]); + } + + if (rb_hash_size(stress_to_class) == 0) { + stress_to_class = 0; + } + } + + return Qnil; +} +#endif + +void * +rb_gc_impl_objspace_alloc(void) +{ + rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t)); + + return objspace; +} + +void +rb_gc_impl_objspace_init(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + gc_config_full_mark_set(TRUE); + + objspace->flags.measure_gc = true; + malloc_limit = gc_params.malloc_limit_min; +#ifdef MALLOC_COUNTERS_NEED_LOCK + rb_native_mutex_initialize(&objspace->malloc_counters.lock); +#endif + objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace); + if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) { + rb_bug("Could not preregister postponed job for GC"); + } + + /* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit + * in at least one pool. In debug builds RVALUE_OVERHEAD can push this + * beyond the 48-byte pool into the 64-byte pool, which is fine. */ + GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]))); + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + + heap->slot_size = pool_slot_sizes[i]; + + ccan_list_head_init(&heap->pages); + } + + init_size_to_heap_idx(); + + rb_darray_make_without_gc(&objspace->heap_pages.sorted, 0); + rb_darray_make_without_gc(&objspace->weak_references, 0); + +#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP) + /* Need to determine if we can use mmap at runtime. */ + heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP; +#endif +#if RGENGC_ESTIMATE_OLDMALLOC + objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min; +#endif + gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES; + + init_mark_stack(&objspace->mark_stack); + + objspace->profile.invoke_time = getrusage_time(); + finalizer_table = st_init_numtable(); +} + +void +rb_gc_impl_init(void) +{ + VALUE gc_constants = rb_hash_new(); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse); + /* Minimum slot size that fits a standard RVALUE */ + size_t rvalue_pool = 0; + for (size_t i = 0; i < HEAP_COUNT; i++) { + if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i]; break; } + } + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(heap_slot_size(HEAP_COUNT - 1))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE)); + if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) { + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue); + } + OBJ_FREEZE(gc_constants); + /* Internal constants in the garbage collector. */ + rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants); + + if (GC_COMPACTION_SUPPORTED) { + rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", gc_verify_compaction_references, -1); + } + else { + rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); + } + +#if GC_DEBUG_STRESS_TO_CLASS + rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1); + rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1); +#endif + + /* internal methods */ + rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0); + +#if MALLOC_ALLOCATED_SIZE + rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0); + rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0); +#endif + + VALUE rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler"); + rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0); + rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0); + rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, 0); + rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0); + rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0); + rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0); + rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1); + rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0); + + { + VALUE opts; + /* \GC build options */ + rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new()); +#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1)) + OPT(GC_DEBUG); + OPT(USE_RGENGC); + OPT(RGENGC_DEBUG); + OPT(RGENGC_CHECK_MODE); + OPT(RGENGC_PROFILE); + OPT(RGENGC_ESTIMATE_OLDMALLOC); + OPT(GC_PROFILE_MORE_DETAIL); + OPT(GC_ENABLE_LAZY_SWEEP); + OPT(CALC_EXACT_MALLOC_SIZE); + OPT(MALLOC_ALLOCATED_SIZE); + OPT(MALLOC_ALLOCATED_SIZE_CHECK); + OPT(GC_PROFILE_DETAIL_MEMORY); + OPT(GC_COMPACTION_SUPPORTED); +#undef OPT + OBJ_FREEZE(opts); + } +} diff --git a/gc/default/extconf.rb b/gc/default/extconf.rb new file mode 100644 index 0000000000..2940a4c962 --- /dev/null +++ b/gc/default/extconf.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "../extconf_base" + +create_gc_makefile("default") diff --git a/gc/extconf_base.rb b/gc/extconf_base.rb new file mode 100644 index 0000000000..2a224b9b0e --- /dev/null +++ b/gc/extconf_base.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "mkmf" + +srcdir = File.join(__dir__, "..") +$INCFLAGS << " -I#{srcdir}" + +$CPPFLAGS << " -DBUILDING_MODULAR_GC" + +append_cflags("-fPIC") + +def create_gc_makefile(name, &block) + create_makefile("librubygc.#{name}", &block) +end diff --git a/gc/gc.h b/gc/gc.h new file mode 100644 index 0000000000..d147a3122a --- /dev/null +++ b/gc/gc.h @@ -0,0 +1,293 @@ +#ifndef GC_GC_H +#define GC_GC_H +/** + * @author Ruby developers <ruby-core@ruby-lang.org> + * @copyright This file is a part of the programming language Ruby. + * Permission is hereby granted, to either redistribute and/or + * modify this file, provided that the conditions mentioned in the + * file COPYING are met. Consult the file for details. + * @brief Private header for the default GC and other GC implementations + * first introduced for [Feature #20470]. + */ +#include "ruby/ruby.h" +#include "ruby/assert.h" + +#include "ruby/thread_native.h" + +#ifndef VM_CHECK_MODE +# define VM_CHECK_MODE RUBY_DEBUG +#endif + +// From ractor_core.h +#ifndef RACTOR_CHECK_MODE +# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE) +#endif + +#if RACTOR_CHECK_MODE +void rb_ractor_setup_belonging(VALUE obj); + +struct rb_gc_obj_suffix { + uint32_t _ractor_belonging_id; +}; + +# define RB_GC_OBJ_HAS_SUFFIX 1 +# define RB_GC_OBJ_SUFFIX_SIZE (sizeof(struct rb_gc_obj_suffix)) +#else +# define RB_GC_OBJ_HAS_SUFFIX 0 +# define RB_GC_OBJ_SUFFIX_SIZE 0 +#endif + +struct rb_gc_vm_context { + rb_nativethread_lock_t lock; + + struct rb_execution_context_struct *ec; +}; + +typedef int (*vm_table_foreach_callback_func)(VALUE value, void *data); +typedef int (*vm_table_update_callback_func)(VALUE *value, void *data); + +enum rb_gc_vm_weak_tables { + RB_GC_VM_CI_TABLE, + RB_GC_VM_OVERLOADED_CME_TABLE, + RB_GC_VM_GLOBAL_SYMBOLS_TABLE, + RB_GC_VM_ID2REF_TABLE, + RB_GC_VM_GENERIC_FIELDS_TABLE, + RB_GC_VM_FROZEN_STRINGS_TABLE, + RB_GC_VM_WEAK_TABLE_COUNT +}; + +#define RB_GC_VM_LOCK() rb_gc_vm_lock(__FILE__, __LINE__) +#define RB_GC_VM_UNLOCK(lev) rb_gc_vm_unlock(lev, __FILE__, __LINE__) +#define RB_GC_CR_LOCK() rb_gc_cr_lock(__FILE__, __LINE__) +#define RB_GC_CR_UNLOCK(lev) rb_gc_cr_unlock(lev, __FILE__, __LINE__) +#define RB_GC_VM_LOCK_NO_BARRIER() rb_gc_vm_lock_no_barrier(__FILE__, __LINE__) +#define RB_GC_VM_UNLOCK_NO_BARRIER(lev) rb_gc_vm_unlock_no_barrier(lev, __FILE__, __LINE__) + +#if USE_MODULAR_GC +# define MODULAR_GC_FN +#else +// This takes advantage of internal linkage winning when appearing first. +// See C99 6.2.2p4. +# define MODULAR_GC_FN static +#endif + +#if USE_MODULAR_GC +RUBY_SYMBOL_EXPORT_BEGIN +#endif + +// These functions cannot be defined as static because they are used by other +// files in Ruby. +size_t rb_size_mul_or_raise(size_t x, size_t y, VALUE exc); +void rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data); +const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj); +const char *rb_obj_info(VALUE obj); +size_t rb_obj_memsize_of(VALUE obj); +bool ruby_free_at_exit_p(void); +void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data); +void rb_gc_verify_shareable(VALUE); + +MODULAR_GC_FN unsigned int rb_gc_vm_lock(const char *file, int line); +MODULAR_GC_FN void rb_gc_vm_unlock(unsigned int lev, const char *file, int line); +MODULAR_GC_FN unsigned int rb_gc_cr_lock(const char *file, int line); +MODULAR_GC_FN void rb_gc_cr_unlock(unsigned int lev, const char *file, int line); +MODULAR_GC_FN unsigned int rb_gc_vm_lock_no_barrier(const char *file, int line); +MODULAR_GC_FN void rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line); +MODULAR_GC_FN void rb_gc_vm_barrier(void); +MODULAR_GC_FN size_t rb_gc_obj_optimal_size(VALUE obj); +MODULAR_GC_FN void rb_gc_mark_children(void *objspace, VALUE obj); +MODULAR_GC_FN void rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, vm_table_update_callback_func update_callback, void *data, bool weak_only, enum rb_gc_vm_weak_tables table); +MODULAR_GC_FN void rb_gc_update_object_references(void *objspace, VALUE obj); +MODULAR_GC_FN void rb_gc_update_vm_references(void *objspace); +MODULAR_GC_FN void rb_gc_event_hook(VALUE obj, rb_event_flag_t event); +MODULAR_GC_FN void *rb_gc_get_objspace(void); +MODULAR_GC_FN void rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data); +MODULAR_GC_FN void rb_gc_set_pending_interrupt(void); +MODULAR_GC_FN void rb_gc_unset_pending_interrupt(void); +MODULAR_GC_FN void rb_gc_obj_free_vm_weak_references(VALUE obj); +MODULAR_GC_FN bool rb_gc_obj_free(void *objspace, VALUE obj); +MODULAR_GC_FN void rb_gc_save_machine_context(void); +MODULAR_GC_FN void rb_gc_mark_roots(void *objspace, const char **categoryp); +MODULAR_GC_FN void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data); +MODULAR_GC_FN bool rb_gc_multi_ractor_p(void); +MODULAR_GC_FN bool rb_gc_shutdown_call_finalizer_p(VALUE obj); +MODULAR_GC_FN void rb_gc_obj_changed_pool(VALUE obj, size_t heap_id); +MODULAR_GC_FN void rb_gc_prepare_heap_process_object(VALUE obj); +MODULAR_GC_FN bool rb_memerror_reentered(void); +MODULAR_GC_FN bool rb_obj_id_p(VALUE); +MODULAR_GC_FN void rb_gc_before_updating_jit_code(void); +MODULAR_GC_FN void rb_gc_after_updating_jit_code(void); +MODULAR_GC_FN bool rb_gc_obj_shareable_p(VALUE); +MODULAR_GC_FN void rb_gc_rp(VALUE); +MODULAR_GC_FN void rb_gc_handle_weak_references(VALUE obj); +MODULAR_GC_FN bool rb_gc_obj_needs_cleanup_p(VALUE obj); + +#if USE_MODULAR_GC +MODULAR_GC_FN bool rb_gc_event_hook_required_p(rb_event_flag_t event); +MODULAR_GC_FN void *rb_gc_get_ractor_newobj_cache(void); +MODULAR_GC_FN void rb_gc_initialize_vm_context(struct rb_gc_vm_context *context); +MODULAR_GC_FN void rb_gc_move_obj_during_marking(VALUE from, VALUE to); +MODULAR_GC_FN void rb_gc_print_backtrace(); +#endif + +#if USE_MODULAR_GC +RUBY_SYMBOL_EXPORT_END +#endif + +void rb_ractor_finish_marking(void); + +// -------------------Private section begin------------------------ +// Functions in this section are private to the default GC and gc.c + +#ifdef BUILDING_MODULAR_GC +RBIMPL_WARNING_PUSH() +RBIMPL_WARNING_IGNORED(-Wunused-function) +#endif + +/* RGENGC_CHECK_MODE + * 0: disable all assertions + * 1: enable assertions (to debug RGenGC) + * 2: enable internal consistency check at each GC (for debugging) + * 3: enable internal consistency check at each GC steps (for debugging) + * 4: enable liveness check + * 5: show all references + */ +#ifndef RGENGC_CHECK_MODE +# define RGENGC_CHECK_MODE 0 +#endif + +#ifndef GC_ASSERT +# define GC_ASSERT(expr, ...) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__)) +#endif + +static int +hash_foreach_replace_value(st_data_t key, st_data_t value, st_data_t argp, int error) +{ + if (rb_gc_location((VALUE)value) != (VALUE)value) { + return ST_REPLACE; + } + return ST_CONTINUE; +} + +static int +hash_replace_ref_value(st_data_t *key, st_data_t *value, st_data_t argp, int existing) +{ + *value = rb_gc_location((VALUE)*value); + + return ST_CONTINUE; +} + +static void +gc_ref_update_table_values_only(st_table *tbl) +{ + if (!tbl || tbl->num_entries == 0) return; + + if (st_foreach_with_replace(tbl, hash_foreach_replace_value, hash_replace_ref_value, 0)) { + rb_raise(rb_eRuntimeError, "hash modified during iteration"); + } +} + +static int +gc_mark_tbl_no_pin_i(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_mark_movable((VALUE)value); + + return ST_CONTINUE; +} + +static int +gc_mark_set_no_pin_i(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_mark_movable((VALUE)key); + + return ST_CONTINUE; +} + +static int +hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error) +{ + if (rb_gc_location((VALUE)key) != (VALUE)key) { + return ST_REPLACE; + } + + if (rb_gc_location((VALUE)value) != (VALUE)value) { + return ST_REPLACE; + } + + return ST_CONTINUE; +} + +static int +hash_replace_ref(st_data_t *key, st_data_t *value, st_data_t argp, int existing) +{ + VALUE new_key = rb_gc_location((VALUE)*key); + if (new_key != (VALUE)*key) { + *key = new_key; + } + + VALUE new_value = rb_gc_location((VALUE)*value); + if (new_value != (VALUE)*value) { + *value = new_value; + } + + return ST_CONTINUE; +} + +static void +gc_update_table_refs(st_table *tbl) +{ + if (!tbl || tbl->num_entries == 0) return; + + if (st_foreach_with_replace(tbl, hash_foreach_replace, hash_replace_ref, 0)) { + rb_raise(rb_eRuntimeError, "hash modified during iteration"); + } +} + +static inline size_t +xmalloc2_size(const size_t count, const size_t elsize) +{ + return rb_size_mul_or_raise(count, elsize, rb_eArgError); +} + +static VALUE +type_sym(size_t type) +{ + switch (type) { +#define COUNT_TYPE(t) case (t): return ID2SYM(rb_intern(#t)); break; + COUNT_TYPE(T_NONE); + COUNT_TYPE(T_OBJECT); + COUNT_TYPE(T_CLASS); + COUNT_TYPE(T_MODULE); + COUNT_TYPE(T_FLOAT); + COUNT_TYPE(T_STRING); + COUNT_TYPE(T_REGEXP); + COUNT_TYPE(T_ARRAY); + COUNT_TYPE(T_HASH); + COUNT_TYPE(T_STRUCT); + COUNT_TYPE(T_BIGNUM); + COUNT_TYPE(T_FILE); + COUNT_TYPE(T_DATA); + COUNT_TYPE(T_MATCH); + COUNT_TYPE(T_COMPLEX); + COUNT_TYPE(T_RATIONAL); + COUNT_TYPE(T_NIL); + COUNT_TYPE(T_TRUE); + COUNT_TYPE(T_FALSE); + COUNT_TYPE(T_SYMBOL); + COUNT_TYPE(T_FIXNUM); + COUNT_TYPE(T_IMEMO); + COUNT_TYPE(T_UNDEF); + COUNT_TYPE(T_NODE); + COUNT_TYPE(T_ICLASS); + COUNT_TYPE(T_ZOMBIE); + COUNT_TYPE(T_MOVED); +#undef COUNT_TYPE + default: return SIZET2NUM(type); break; + } +} + +#ifdef BUILDING_MODULAR_GC +RBIMPL_WARNING_POP() +#endif +// -------------------Private section end------------------------ + +#endif diff --git a/gc/gc_impl.h b/gc/gc_impl.h new file mode 100644 index 0000000000..d9e44cc66d --- /dev/null +++ b/gc/gc_impl.h @@ -0,0 +1,127 @@ +#ifndef GC_GC_IMPL_H +#define GC_GC_IMPL_H +/** + * @author Ruby developers <ruby-core@ruby-lang.org> + * @copyright This file is a part of the programming language Ruby. + * Permission is hereby granted, to either redistribute and/or + * modify this file, provided that the conditions mentioned in the + * file COPYING are met. Consult the file for details. + * @brief Header for GC implementations introduced in [Feature #20470]. + */ +#include "ruby/ruby.h" + +#ifndef RB_GC_OBJECT_METADATA_ENTRY_DEFINED +# define RB_GC_OBJECT_METADATA_ENTRY_DEFINED +struct rb_gc_object_metadata_entry { + ID name; + VALUE val; +}; +#endif + +#ifdef BUILDING_MODULAR_GC +# define GC_IMPL_FN +#else +// `GC_IMPL_FN` is an implementation detail of `!USE_MODULAR_GC` builds +// to have the default GC in the same translation unit as gc.c for +// the sake of optimizer visibility. It expands to nothing unless +// you're the default GC. +// +// For the default GC, do not copy-paste this when implementing +// these functions. This takes advantage of internal linkage winning +// when appearing first. See C99 6.2.2p4. +# define GC_IMPL_FN static +#endif + +// Bootup +GC_IMPL_FN void *rb_gc_impl_objspace_alloc(void); +GC_IMPL_FN void rb_gc_impl_objspace_init(void *objspace_ptr); +GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor); +GC_IMPL_FN void rb_gc_impl_set_params(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_init(void); +GC_IMPL_FN size_t *rb_gc_impl_heap_sizes(void *objspace_ptr); +// Shutdown +GC_IMPL_FN void rb_gc_impl_shutdown_free_objects(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache); +// GC +GC_IMPL_FN void rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact); +GC_IMPL_FN bool rb_gc_impl_during_gc_p(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_prepare_heap(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_gc_enable(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc); +GC_IMPL_FN bool rb_gc_impl_gc_enabled_p(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag); +GC_IMPL_FN VALUE rb_gc_impl_stress_get(void *objspace_ptr); +GC_IMPL_FN VALUE rb_gc_impl_config_get(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_config_set(void *objspace_ptr, VALUE hash); +GC_IMPL_FN struct rb_gc_vm_context *rb_gc_impl_get_vm_context(void *objspace_ptr); +// Object allocation +GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size); +GC_IMPL_FN size_t rb_gc_impl_obj_slot_size(VALUE obj); +GC_IMPL_FN size_t rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size); +GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size); +// Malloc +/* + * BEWARE: These functions may or may not run under GVL. + * + * You might want to make them thread-safe. + * Garbage collecting inside is possible if and only if you + * already have GVL. Also raising exceptions without one is a + * total disaster. + * + * When you absolutely cannot allocate the requested amount of + * memory just return NULL (with appropriate errno set). + * The caller side takes care of that situation. + */ +GC_IMPL_FN void *rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed); +GC_IMPL_FN void *rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed); +GC_IMPL_FN void *rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed); +GC_IMPL_FN void rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size); +GC_IMPL_FN void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff); +// Marking +GC_IMPL_FN void rb_gc_impl_mark(void *objspace_ptr, VALUE obj); +GC_IMPL_FN void rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr); +GC_IMPL_FN void rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj); +GC_IMPL_FN void rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj); +// Weak references +GC_IMPL_FN void rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj); +GC_IMPL_FN bool rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj); +// Compaction +GC_IMPL_FN void rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj); +GC_IMPL_FN bool rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj); +GC_IMPL_FN VALUE rb_gc_impl_location(void *objspace_ptr, VALUE value); +// Write barriers +GC_IMPL_FN void rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b); +GC_IMPL_FN void rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj); +GC_IMPL_FN void rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj); +// Heap walking +GC_IMPL_FN void rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); +GC_IMPL_FN void rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data); +// Finalizers +GC_IMPL_FN void rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data); +GC_IMPL_FN VALUE rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block); +GC_IMPL_FN void rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj); +GC_IMPL_FN void rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj); +GC_IMPL_FN void rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr); +// Forking +GC_IMPL_FN void rb_gc_impl_before_fork(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid); +// Statistics +GC_IMPL_FN void rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag); +GC_IMPL_FN bool rb_gc_impl_get_measure_total_time(void *objspace_ptr); +GC_IMPL_FN unsigned long long rb_gc_impl_get_total_time(void *objspace_ptr); +GC_IMPL_FN size_t rb_gc_impl_gc_count(void *objspace_ptr); +GC_IMPL_FN VALUE rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key); +GC_IMPL_FN VALUE rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym); +GC_IMPL_FN VALUE rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym); +GC_IMPL_FN const char *rb_gc_impl_active_gc_name(void); +// Miscellaneous +GC_IMPL_FN struct rb_gc_object_metadata_entry *rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj); +GC_IMPL_FN bool rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr); +GC_IMPL_FN bool rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj); +GC_IMPL_FN void rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event); +GC_IMPL_FN void rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj); + +#undef GC_IMPL_FN + +#endif diff --git a/gc/mmtk/.gitignore b/gc/mmtk/.gitignore new file mode 100644 index 0000000000..eb5a316cbd --- /dev/null +++ b/gc/mmtk/.gitignore @@ -0,0 +1 @@ +target diff --git a/gc/mmtk/Cargo.lock b/gc/mmtk/Cargo.lock new file mode 100644 index 0000000000..910048fa80 --- /dev/null +++ b/gc/mmtk/Cargo.lock @@ -0,0 +1,1108 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-traits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f750b93bd1b739cf9ddf85f8fe7c97a4a62c60ccf8b6f232514bd9103bedc" +dependencies = [ + "cfg-if", + "rustc_version", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" +dependencies = [ + "git2", +] + +[[package]] +name = "bytemuck" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "cc" +version = "1.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "delegate" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b6483c2bbed26f97861cf57651d4f2b731964a28cd2257f934a4b452480d21" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "downcast-rs" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279259b0ac81c89d11c290495fdcfa96ea3643b7df311c138b6fe8ca5237f0f8" +dependencies = [ + "idna_mapping", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna_mapping" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c13906586a4b339310541a274dd927aff6fcbb5b8e3af90634c4b31681c792" +dependencies = [ + "unicode-joining-type", +] + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.1", + "libc", + "windows-sys", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "jiff" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libgit2-sys" +version = "0.18.3+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mmtk" +version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791" +dependencies = [ + "atomic", + "atomic-traits", + "atomic_refcell", + "built", + "bytemuck", + "bytemuck_derive", + "cfg-if", + "crossbeam", + "delegate", + "downcast-rs", + "enum-map", + "env_logger", + "idna_adapter", + "is-terminal", + "itertools", + "lazy_static", + "libc", + "log", + "memoffset", + "mmtk-macros", + "num-traits", + "num_cpus", + "portable-atomic", + "probe", + "rayon-core", + "regex", + "rustversion", + "spin", + "static_assertions", + "strum", + "strum_macros", + "sysinfo 0.33.1", +] + +[[package]] +name = "mmtk-macros" +version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "mmtk_ruby" +version = "0.1.0" +dependencies = [ + "atomic_refcell", + "env_logger", + "libc", + "log", + "mmtk", + "once_cell", + "probe", + "sysinfo 0.32.1", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "probe" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e2d2444b730c8f027344c60f9e1f1554d7a3342df9bdd425142ed119a6e5a3" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" + +[[package]] +name = "strum_macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.101", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-joining-type" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d00a78170970967fdb83f9d49b92f959ab2bb829186b113e4f4604ad98e180" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] diff --git a/gc/mmtk/Cargo.toml b/gc/mmtk/Cargo.toml new file mode 100644 index 0000000000..d856122900 --- /dev/null +++ b/gc/mmtk/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "mmtk_ruby" +version = "0.1.0" +authors = [] +edition = "2021" + +[lib] +name = "mmtk_ruby" +crate-type = ["cdylib", "staticlib"] + +[profile.release] +lto = true + +[dependencies] +libc = "0.2" +log = "0.4.14" +env_logger = "0.11.3" +once_cell = "1.17.0" +atomic_refcell = "0.1.9" +probe = "0.5" +sysinfo = "0.32.0" + +[dependencies.mmtk] +features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"] + +# Uncomment the following lines to use mmtk-core from the official repository. +git = "https://github.com/mmtk/mmtk-core.git" +rev = "c6317a3f1c262e33fc2e427e4cc999c17bcc4791" + +# Uncomment the following line to use mmtk-core from a local repository. +# path = "../../../mmtk-core" + +[features] +default = [] + +# When moving an object, clear its original copy. +clear_old_copy = [] + +# Enable extra assertions in release build. For debugging. +extra_assert = [] + +[workspace] diff --git a/gc/mmtk/cbindgen.toml b/gc/mmtk/cbindgen.toml new file mode 100644 index 0000000000..b99c30efc8 --- /dev/null +++ b/gc/mmtk/cbindgen.toml @@ -0,0 +1,36 @@ +language = "C" + +include_guard = "MMTK_H" + +autogen_warning = "/* Warning, this file is autogenerated by cbindgen from the mmtk-ruby repository. Don't modify this manually. */" + +tab_width = 4 + +usize_is_size_t = true + +after_includes = """ + +typedef struct MMTk_Builder MMTk_Builder; +typedef struct MMTk_Mutator MMTk_Mutator; + +typedef struct MMTk_ractor_cache *MMTk_VMThread; +typedef struct MMTk_ractor_cache *MMTk_VMMutatorThread; +typedef struct MMTk_GCThreadTLS *MMTk_VMWorkerThread; +typedef void *MMTk_Address; +typedef void *MMTk_ObjectReference; +typedef void *MMTk_NullableObjectReference; +typedef uint32_t MMTk_AllocationSemantics; + +typedef struct MMTk_BumpPointer { + uintptr_t cursor; + uintptr_t limit; +} MMTk_BumpPointer; +""" + +[export] +exclude = ["RubyMutator"] +prefix = "MMTk_" + +[export.rename] +"MMTKBuilder" = "Builder" +"RubyMutator" = "Mutator" diff --git a/gc/mmtk/depend b/gc/mmtk/depend new file mode 100644 index 0000000000..77b229af36 --- /dev/null +++ b/gc/mmtk/depend @@ -0,0 +1,18 @@ +$(TARGET_SO): $(MMTK_BUILD)/$(LIBMMTK_RUBY) + +# Add the `libmmtk_ruby.a` target to run `cargo build` + +release/$(LIBMMTK_RUBY) debug/$(LIBMMTK_RUBY): $(RUSTSRCS) $(srcdir)/Cargo.toml $(srcdir)/Cargo.toml + +release/$(LIBMMTK_RUBY): + CARGO_TARGET_DIR="." cargo build --manifest-path=$(srcdir)/Cargo.toml --release + +debug/$(LIBMMTK_RUBY): + CARGO_TARGET_DIR="." cargo build --manifest-path=$(srcdir)/Cargo.toml + +clean: clean-mmtk + +.PHONY: clean-mmtk +clean-mmtk: + -$(Q)$(RM_RF) debug release + -$(Q)$(RM) .rustc_info.json diff --git a/gc/mmtk/extconf.rb b/gc/mmtk/extconf.rb new file mode 100644 index 0000000000..c0e788037e --- /dev/null +++ b/gc/mmtk/extconf.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require_relative "../extconf_base" + +# Statically link `libmmtk_ruby.a` +$LIBS << " $(MMTK_BUILD)/$(LIBMMTK_RUBY)" + +rustsrcs = Dir.glob("src/*.rs", base: __dir__).map {|s| "$(srcdir)/#{s}"} + +create_gc_makefile("mmtk") do |makefile| + [ + *makefile, + + <<~MAKEFILE, + MMTK_BUILD = debug + LIBMMTK_RUBY = libmmtk_ruby.#$LIBEXT + RUSTSRCS = #{rustsrcs.join(" \\\n\t ")} + + ifeq ($(MMTK_BUILD), debug) + CPPFLAGS += -DMMTK_DEBUG + endif + MAKEFILE + ] +end diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c new file mode 100644 index 0000000000..8be69b4fe6 --- /dev/null +++ b/gc/mmtk/mmtk.c @@ -0,0 +1,1658 @@ +#include <pthread.h> +#include <stdbool.h> + +#include "ruby/assert.h" +#include "ruby/atomic.h" +#include "ruby/debug.h" + +#include "gc/gc.h" +#include "gc/gc_impl.h" +#include "gc/mmtk/mmtk.h" + +#include "ccan/list/list.h" +#include "darray.h" + +#ifdef __APPLE__ +#include <sys/sysctl.h> +#endif + +struct objspace { + bool measure_gc_time; + bool gc_stress; + + size_t gc_count; + size_t moving_gc_count; + size_t total_gc_time; + size_t total_allocated_objects; + + st_table *finalizer_table; + struct MMTk_final_job *finalizer_jobs; + rb_postponed_job_handle_t finalizer_postponed_job; + + struct ccan_list_head ractor_caches; + unsigned long live_ractor_cache_count; + + pthread_mutex_t mutex; + rb_atomic_t mutator_blocking_count; + bool world_stopped; + pthread_cond_t cond_world_stopped; + pthread_cond_t cond_world_started; + size_t start_the_world_count; + + pthread_mutex_t event_hook_mutex; + + struct { + bool gc_thread_crashed; + char crash_msg[256]; + } crash_context; + + struct rb_gc_vm_context vm_context; + + unsigned int fork_hook_vm_lock_lev; +}; + +#define OBJ_FREE_BUF_CAPACITY 128 + +struct MMTk_ractor_cache { + struct ccan_list_node list_node; + + MMTk_Mutator *mutator; + bool gc_mutator_p; + + MMTk_BumpPointer *bump_pointer; + + MMTk_ObjectReference obj_free_parallel_buf[OBJ_FREE_BUF_CAPACITY]; + size_t obj_free_parallel_count; + MMTk_ObjectReference obj_free_non_parallel_buf[OBJ_FREE_BUF_CAPACITY]; + size_t obj_free_non_parallel_count; +}; + +struct MMTk_final_job { + struct MMTk_final_job *next; + enum { + MMTK_FINAL_JOB_DFREE, + MMTK_FINAL_JOB_FINALIZE, + } kind; + union { + struct { + void (*func)(void *); + void *data; + } dfree; + struct { + /* HACK: we store the object ID on the 0th element of this array. */ + VALUE finalizer_array; + } finalize; + } as; +}; + +#ifdef RB_THREAD_LOCAL_SPECIFIER +RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls; + +RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object; +#else +# error We currently need language-supported TLS +#endif + +#ifdef MMTK_DEBUG +# define MMTK_ASSERT(expr, ...) RUBY_ASSERT_ALWAYS(expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__)) +#else +# define MMTK_ASSERT(expr, ...) ((void)0) +#endif + +#include <pthread.h> + +static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); + +static void +rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls) +{ + rb_mmtk_gc_thread_tls = gc_thread_tls; +} + +static bool +rb_mmtk_is_mutator(void) +{ + return ruby_native_thread_p(); +} + +static void +rb_mmtk_stop_the_world(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + while (!objspace->world_stopped) { + pthread_cond_wait(&objspace->cond_world_stopped, &objspace->mutex); + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void +rb_mmtk_resume_mutators(bool current_gc_may_move) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + objspace->world_stopped = false; + objspace->gc_count++; + if (current_gc_may_move) objspace->moving_gc_count++; + pthread_cond_broadcast(&objspace->cond_world_started); + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache); + +static void +rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + size_t starting_gc_count = objspace->gc_count; + RUBY_ATOMIC_INC(objspace->mutator_blocking_count); + int lock_lev = RB_GC_VM_LOCK(); + RUBY_ATOMIC_DEC(objspace->mutator_blocking_count); + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + if (objspace->gc_count == starting_gc_count) { + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START); + + rb_gc_initialize_vm_context(&objspace->vm_context); + + mutator->gc_mutator_p = true; + + struct timespec gc_start_time; + if (objspace->measure_gc_time) { + clock_gettime(CLOCK_MONOTONIC, &gc_start_time); + } + + rb_gc_save_machine_context(); + + rb_gc_vm_barrier(); + + struct MMTk_ractor_cache *rc; + ccan_list_for_each(&objspace->ractor_caches, rc, list_node) { + mmtk_flush_obj_free_buffer(rc); + } + + objspace->world_stopped = true; + + pthread_cond_broadcast(&objspace->cond_world_stopped); + + // Wait for GC end + while (objspace->world_stopped) { + pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex); + } + + if (RB_UNLIKELY(objspace->crash_context.gc_thread_crashed)) { + rb_bug("%s", objspace->crash_context.crash_msg); + } + + if (objspace->measure_gc_time) { + struct timespec gc_end_time; + clock_gettime(CLOCK_MONOTONIC, &gc_end_time); + + objspace->total_gc_time += + (gc_end_time.tv_sec - gc_start_time.tv_sec) * (1000 * 1000 * 1000) + + (gc_end_time.tv_nsec - gc_start_time.tv_nsec); + } + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } + RB_GC_VM_UNLOCK(lock_lev); +} + +static void +rb_mmtk_before_updating_jit_code(void) +{ + rb_gc_before_updating_jit_code(); +} + +static void +rb_mmtk_after_updating_jit_code(void) +{ + rb_gc_after_updating_jit_code(); +} + +static size_t +rb_mmtk_number_of_mutators(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + return objspace->live_ractor_cache_count; +} + +static void +rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data) +{ + struct objspace *objspace = rb_gc_get_objspace(); + struct MMTk_ractor_cache *ractor_cache; + + ccan_list_for_each(&objspace->ractor_caches, ractor_cache, list_node) { + visit_mutator(ractor_cache->mutator, data); + } +} + +static void +rb_mmtk_scan_gc_roots(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + rb_gc_mark_roots(objspace, NULL); +} + +static int +pin_value(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); + + return ST_CONTINUE; +} + +static void +rb_mmtk_scan_objspace(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + if (objspace->finalizer_table != NULL) { + st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace); + } + + struct MMTk_final_job *job = objspace->finalizer_jobs; + while (job != NULL) { + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + break; + case MMTK_FINAL_JOB_FINALIZE: + rb_gc_impl_mark(objspace, job->as.finalize.finalizer_array); + break; + default: + rb_bug("rb_mmtk_scan_objspace: unknown final job type %d", job->kind); + } + + job = job->next; + } +} + +static void +rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to) +{ + rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to); +} + +static void +rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object) +{ + VALUE object = (VALUE)mmtk_object; + + if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) { + marking_parent_object = object; + rb_gc_update_object_references(rb_gc_get_objspace(), object); + marking_parent_object = 0; + } +} + +static void +rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object) +{ + marking_parent_object = (VALUE)object; + rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object); + marking_parent_object = 0; +} + +static void +rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving) +{ + VALUE object = (VALUE)mmtk_object; + + marking_parent_object = object; + + rb_gc_handle_weak_references(object); + + if (moving) { + rb_gc_update_object_references(rb_gc_get_objspace(), object); + } + + marking_parent_object = 0; +} + +static void +rb_mmtk_call_obj_free(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + struct objspace *objspace = rb_gc_get_objspace(); + + if (RB_UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_FREEOBJ))) { + pthread_mutex_lock(&objspace->event_hook_mutex); + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); + pthread_mutex_unlock(&objspace->event_hook_mutex); + } + + if (RB_UNLIKELY(rb_gc_obj_needs_cleanup_p(obj))) { + rb_gc_obj_free(objspace, obj); + } + +#ifdef MMTK_DEBUG + memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj)); +#endif +} + +static size_t +rb_mmtk_vm_live_bytes(void) +{ + return 0; +} + +static void +make_final_job(struct objspace *objspace, VALUE obj, VALUE table) +{ + MMTK_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY); + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->next = objspace->finalizer_jobs; + job->kind = MMTK_FINAL_JOB_FINALIZE; + job->as.finalize.finalizer_array = table; + + objspace->finalizer_jobs = job; +} + +static int +rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data, int error) +{ + MMTK_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value)); + MMTK_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY); + + struct objspace *objspace = (struct objspace *)data; + + if (mmtk_is_reachable((MMTk_ObjectReference)key)) { + VALUE new_key_location = rb_mmtk_call_object_closure((VALUE)key, false); + + MMTK_ASSERT(RB_FL_TEST(new_key_location, RUBY_FL_FINALIZE)); + + if (new_key_location != key) { + return ST_REPLACE; + } + } + else { + make_final_job(objspace, (VALUE)key, (VALUE)value); + + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static int +rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_data_t data, int existing) +{ + *key = rb_mmtk_call_object_closure((VALUE)*key, false); + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_finalizer_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + st_foreach_with_replace( + objspace->finalizer_table, + rb_mmtk_update_finalizer_table_i, + rb_mmtk_update_finalizer_table_replace_i, + (st_data_t)objspace + ); +} + +static int +rb_mmtk_global_tables_count(void) +{ + return RB_GC_VM_WEAK_TABLE_COUNT; +} + +static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); + +static int +rb_mmtk_update_global_tables_i(VALUE val, void *data) +{ + if (!mmtk_is_reachable((MMTk_ObjectReference)val)) { + return ST_DELETE; + } + + // TODO: check only if in moving GC + if (rb_mmtk_call_object_closure(val, false) != val) { + return ST_REPLACE; + } + + return ST_CONTINUE; +} + +static int +rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data) +{ + // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice + *ptr = rb_mmtk_call_object_closure(*ptr, false); + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_global_tables(int table, bool moving) +{ + MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT); + + rb_gc_vm_weak_table_foreach( + rb_mmtk_update_global_tables_i, + rb_mmtk_update_global_tables_replace_i, + NULL, + !moving, + (enum rb_gc_vm_weak_tables)table + ); +} + +static bool +rb_mmtk_special_const_p(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + + return RB_SPECIAL_CONST_P(obj); +} + +RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2) +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_gc_thread_bug(const char *msg, ...) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + objspace->crash_context.gc_thread_crashed = true; + + va_list args; + va_start(args, msg); + vsnprintf(objspace->crash_context.crash_msg, sizeof(objspace->crash_context.crash_msg), msg, args); + va_end(args); + + fprintf(stderr, "-- GC thread backtrace " + "-------------------------------------------\n"); + rb_gc_print_backtrace(); + fprintf(stderr, "\n"); + + rb_mmtk_resume_mutators(false); + + sleep(5); + + rb_bug("rb_mmtk_gc_thread_bug"); +} + +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_gc_thread_panic_handler(void) +{ + rb_mmtk_gc_thread_bug("MMTk GC thread panicked"); +} + +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_mutator_thread_panic_handler(void) +{ + rb_bug("Ruby mutator thread panicked"); +} + +// Bootup +MMTk_RubyUpcalls ruby_upcalls = { + rb_mmtk_init_gc_worker_thread, + rb_mmtk_is_mutator, + rb_mmtk_stop_the_world, + rb_mmtk_resume_mutators, + rb_mmtk_block_for_gc, + rb_mmtk_before_updating_jit_code, + rb_mmtk_after_updating_jit_code, + rb_mmtk_number_of_mutators, + rb_mmtk_get_mutators, + rb_mmtk_scan_gc_roots, + rb_mmtk_scan_objspace, + rb_mmtk_move_obj_during_marking, + rb_mmtk_update_object_references, + rb_mmtk_call_gc_mark_children, + rb_mmtk_handle_weak_references, + rb_mmtk_call_obj_free, + rb_mmtk_vm_live_bytes, + rb_mmtk_update_global_tables, + rb_mmtk_global_tables_count, + rb_mmtk_update_finalizer_table, + rb_mmtk_special_const_p, + rb_mmtk_mutator_thread_panic_handler, + rb_mmtk_gc_thread_panic_handler, +}; + +// Use max 80% of the available memory by default for MMTk +#define RB_MMTK_HEAP_LIMIT_PERC 80 +#define RB_MMTK_DEFAULT_HEAP_MIN (1024 * 1024) +#define RB_MMTK_DEFAULT_HEAP_MAX (rb_mmtk_system_physical_memory() / 100 * RB_MMTK_HEAP_LIMIT_PERC) + +enum mmtk_heap_mode { + RB_MMTK_DYNAMIC_HEAP, + RB_MMTK_FIXED_HEAP +}; + +MMTk_Builder * +rb_mmtk_builder_init(void) +{ + MMTk_Builder *builder = mmtk_builder_default(); + return builder; +} + +void * +rb_gc_impl_objspace_alloc(void) +{ + MMTk_Builder *builder = rb_mmtk_builder_init(); + MMTk_RubyBindingOptions binding_options = { + .suffix_size = RB_GC_OBJ_SUFFIX_SIZE, + }; + mmtk_init_binding(builder, &binding_options, &ruby_upcalls); + + return calloc(1, sizeof(struct objspace)); +} + +static void gc_run_finalizers(void *data); + +void +rb_gc_impl_objspace_init(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + objspace->measure_gc_time = true; + + objspace->finalizer_table = st_init_numtable(); + objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace); + + ccan_list_head_init(&objspace->ractor_caches); + + objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; + objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + + objspace->event_hook_mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; +} + +void +rb_gc_impl_objspace_free(void *objspace_ptr) +{ + free(objspace_ptr); +} + +void * +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) +{ + struct objspace *objspace = objspace_ptr; + if (objspace->live_ractor_cache_count == 0) { + mmtk_initialize_collection(ractor); + } + objspace->live_ractor_cache_count++; + + struct MMTk_ractor_cache *cache = calloc(1, sizeof(struct MMTk_ractor_cache)); + ccan_list_add(&objspace->ractor_caches, &cache->list_node); + + cache->mutator = mmtk_bind_mutator(cache); + cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator); + + return cache; +} + +void +rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache_ptr) +{ + struct objspace *objspace = objspace_ptr; + struct MMTk_ractor_cache *cache = cache_ptr; + + ccan_list_del(&cache->list_node); + + mmtk_flush_obj_free_buffer(cache); + + if (ruby_free_at_exit_p()) { + MMTK_ASSERT(objspace->live_ractor_cache_count > 0); + } + else { + MMTK_ASSERT(objspace->live_ractor_cache_count > 1); + } + + objspace->live_ractor_cache_count--; + + mmtk_destroy_mutator(cache->mutator); +} + +void rb_gc_impl_set_params(void *objspace_ptr) { } + +static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; } + +#if SIZEOF_VALUE >= 8 +#define MMTK_HEAP_COUNT 12 +#define MMTK_MAX_OBJ_SIZE 1024 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 40, 64, 80, 96, 128, 160, 256, 512, 640, 768, MMTK_MAX_OBJ_SIZE, 0 +}; +#else +#define MMTK_HEAP_COUNT 5 +#define MMTK_MAX_OBJ_SIZE 512 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0 +}; +#endif + +void +rb_gc_impl_init(void) +{ + VALUE gc_constants = rb_hash_new(); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); + // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0)); + OBJ_FREEZE(gc_constants); + rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants); + + // no-ops for compatibility + rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0); + + rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); +} + +size_t * +rb_gc_impl_heap_sizes(void *objspace_ptr) +{ + return heap_sizes; +} + +int +rb_mmtk_obj_free_iter_wrapper(VALUE obj, void *data) +{ + struct objspace *objspace = data; + + if (!RB_TYPE_P(obj, T_NONE)) { + rb_gc_obj_free_vm_weak_references(obj); + rb_gc_obj_free(objspace, obj); + } + + return 0; +} + +// Shutdown +static void each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data); + +void +rb_gc_impl_shutdown_free_objects(void *objspace_ptr) +{ + mmtk_set_gc_enabled(false); + each_object(objspace_ptr, rb_mmtk_obj_free_iter_wrapper, objspace_ptr); + mmtk_set_gc_enabled(true); +} + +// GC +void +rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact) +{ + mmtk_handle_user_collection_request(rb_gc_get_ractor_newobj_cache(), true, full_mark); +} + +bool +rb_gc_impl_during_gc_p(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + return objspace->world_stopped; +} + +static void +rb_gc_impl_prepare_heap_i(MMTk_ObjectReference obj, void *d) +{ + rb_gc_prepare_heap_process_object((VALUE)obj); +} + +void +rb_gc_impl_prepare_heap(void *objspace_ptr) +{ + mmtk_enumerate_objects(rb_gc_impl_prepare_heap_i, NULL); +} + +void +rb_gc_impl_gc_enable(void *objspace_ptr) +{ + mmtk_set_gc_enabled(true); +} + +void +rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc) +{ + mmtk_set_gc_enabled(false); +} + +bool +rb_gc_impl_gc_enabled_p(void *objspace_ptr) +{ + return mmtk_gc_enabled_p(); +} + +void +rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag) +{ + struct objspace *objspace = objspace_ptr; + + objspace->gc_stress = RTEST(flag); +} + +VALUE +rb_gc_impl_stress_get(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->gc_stress ? Qtrue : Qfalse; +} + +VALUE +rb_gc_impl_config_get(void *objspace_ptr) +{ + VALUE hash = rb_hash_new(); + + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_worker_count")), RB_ULONG2NUM(mmtk_worker_count())); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_plan")), rb_str_new_cstr((const char *)mmtk_plan())); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_mode")), rb_str_new_cstr((const char *)mmtk_heap_mode())); + size_t heap_min = mmtk_heap_min(); + if (heap_min > 0) rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_min")), RB_ULONG2NUM(heap_min)); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_max")), RB_ULONG2NUM(mmtk_heap_max())); + + return hash; +} + +void +rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) +{ + // TODO +} + +struct rb_gc_vm_context * +rb_gc_impl_get_vm_context(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return &objspace->vm_context; +} + +// Object allocation + +static VALUE +rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size, size_t align) +{ + MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer; + if (bump_pointer == NULL) return 0; + + uintptr_t cursor = bump_pointer->cursor; + + // Ensure cursor is aligned + size_t mask = align - 1; + cursor = (cursor + mask) & ~mask; + + cursor += size; + + if (cursor > bump_pointer->limit) { + return 0; + } + else { + VALUE obj = cursor - size; + bump_pointer->cursor = cursor; + return obj; + } +} + +static bool +obj_can_parallel_free_p(VALUE obj) +{ + switch (RB_BUILTIN_TYPE(obj)) { + case T_ARRAY: + case T_BIGNUM: + case T_COMPLEX: + case T_FLOAT: + case T_HASH: + case T_OBJECT: + case T_RATIONAL: + case T_REGEXP: + case T_STRING: + case T_STRUCT: + case T_SYMBOL: + return true; + default: + return false; + } +} + +static void +mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache) +{ + if (cache->obj_free_parallel_count > 0) { + mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf, + cache->obj_free_parallel_count, true); + cache->obj_free_parallel_count = 0; + } + if (cache->obj_free_non_parallel_count > 0) { + mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf, + cache->obj_free_non_parallel_count, false); + cache->obj_free_non_parallel_count = 0; + } +} + +static inline void +mmtk_buffer_obj_free_candidate(struct MMTk_ractor_cache *cache, VALUE obj) +{ + if (obj_can_parallel_free_p(obj)) { + cache->obj_free_parallel_buf[cache->obj_free_parallel_count++] = (MMTk_ObjectReference)obj; + if (cache->obj_free_parallel_count >= OBJ_FREE_BUF_CAPACITY) { + mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf, + cache->obj_free_parallel_count, true); + cache->obj_free_parallel_count = 0; + } + } + else { + cache->obj_free_non_parallel_buf[cache->obj_free_non_parallel_count++] = (MMTk_ObjectReference)obj; + if (cache->obj_free_non_parallel_count >= OBJ_FREE_BUF_CAPACITY) { + mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf, + cache->obj_free_non_parallel_count, false); + cache->obj_free_non_parallel_count = 0; + } + } +} + +VALUE +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) +{ +#define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0 + struct objspace *objspace = objspace_ptr; + struct MMTk_ractor_cache *ractor_cache = cache_ptr; + + if (alloc_size > MMTK_MAX_OBJ_SIZE) rb_bug("too big"); + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + if (alloc_size == heap_sizes[i]) break; + if (alloc_size < heap_sizes[i]) { + alloc_size = heap_sizes[i]; + break; + } + } + + if (objspace->gc_stress) { + mmtk_handle_user_collection_request(ractor_cache, false, false); + } + + // Layout: [hidden size header (sizeof(VALUE))][payload (alloc_size)][suffix (RB_GC_OBJ_SUFFIX_SIZE)] + alloc_size += sizeof(VALUE) + RB_GC_OBJ_SUFFIX_SIZE; + + VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN); + if (!alloc_obj) { + alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT); + } + + alloc_obj++; + alloc_obj[-1] = alloc_size - sizeof(VALUE) - RB_GC_OBJ_SUFFIX_SIZE; + alloc_obj[0] = flags; + alloc_obj[1] = klass; + + // TODO: implement fast path for mmtk_post_alloc + mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size, MMTK_ALLOCATION_SEMANTICS_DEFAULT); + + // TODO: only add when object needs obj_free to be called + mmtk_buffer_obj_free_candidate(ractor_cache, (VALUE)alloc_obj); + + objspace->total_allocated_objects++; + + return (VALUE)alloc_obj; +} + +size_t +rb_gc_impl_obj_slot_size(VALUE obj) +{ + return ((VALUE *)obj)[-1]; +} + +size_t +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) +{ + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + if (size == heap_sizes[i]) return i; + if (size < heap_sizes[i]) return i; + } + + rb_bug("size too big"); +} + +bool +rb_gc_impl_size_allocatable_p(size_t size) +{ + return size <= MMTK_MAX_OBJ_SIZE; +} + +// Malloc +void * +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + // TODO: don't use system malloc + return malloc(size); +} + +void * +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + // TODO: don't use system calloc + return calloc(1, size); +} + +void * +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) +{ + // TODO: don't use system realloc + return realloc(ptr, new_size); +} + +void +rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) +{ + // TODO: don't use system free + free(ptr); +} + +void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { } + +// Marking +static inline VALUE +rb_mmtk_call_object_closure(VALUE obj, bool pin) +{ + if (RB_UNLIKELY(RB_BUILTIN_TYPE(obj) == T_NONE)) { + enum { info_size = 256 }; + char obj_info_buf[info_size]; + rb_raw_obj_info(obj_info_buf, info_size, obj); + + char parent_obj_info_buf[info_size]; + rb_raw_obj_info(parent_obj_info_buf, info_size, marking_parent_object); + + rb_mmtk_gc_thread_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf); + } + + return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function( + rb_mmtk_gc_thread_tls->object_closure.rust_closure, + rb_mmtk_gc_thread_tls->gc_context, + (MMTk_ObjectReference)obj, + pin + ); +} + +void +rb_gc_impl_mark(void *objspace_ptr, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_call_object_closure(obj, false); +} + +void +rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) +{ + if (RB_SPECIAL_CONST_P(*ptr)) return; + + VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false); + if (new_obj != *ptr) { + *ptr = new_obj; + } +} + +void +rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_call_object_closure(obj, true); +} + +void +rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) +{ + if (rb_gc_impl_pointer_to_heap_p(objspace_ptr, (const void *)obj)) { + rb_gc_impl_mark_and_pin(objspace_ptr, obj); + } +} + +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) +{ + RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE); + mmtk_declare_weak_references((MMTk_ObjectReference)obj); +} + +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + return mmtk_weak_references_alive_p((MMTk_ObjectReference)obj); +} + +// Compaction +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + mmtk_register_pinning_obj((MMTk_ObjectReference)obj); +} + +bool +rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) +{ + return rb_mmtk_call_object_closure(obj, false) != obj; +} + +VALUE +rb_gc_impl_location(void *objspace_ptr, VALUE obj) +{ + return rb_mmtk_call_object_closure(obj, false); +} + +// Write barriers +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache(); + + if (SPECIAL_CONST_P(b)) return; + +#ifdef MMTK_DEBUG + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)a)) { + char buff[256]; + rb_bug("a: %s is not an object", rb_raw_obj_info(buff, 256, a)); + } + + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)b)) { + char buff[256]; + rb_bug("b: %s is not an object", rb_raw_obj_info(buff, 256, b)); + } +#endif + + MMTK_ASSERT(BUILTIN_TYPE(a) != T_NONE); + MMTK_ASSERT(BUILTIN_TYPE(b) != T_NONE); + + mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)a); +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj); +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache(); + + mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)obj); +} + +// Heap walking +static void +each_objects_i(MMTk_ObjectReference obj, void *d) +{ + rb_darray(VALUE) *objs = d; + + rb_darray_append(objs, (VALUE)obj); +} + +static void +each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data) +{ + rb_darray(VALUE) objs; + rb_darray_make(&objs, 0); + + mmtk_enumerate_objects(each_objects_i, &objs); + + VALUE *obj_ptr; + rb_darray_foreach(objs, i, obj_ptr) { + if (!mmtk_is_mmtk_object((MMTk_ObjectReference)*obj_ptr)) continue; + + if (func(*obj_ptr, data) != 0) { + break; + } + } + + rb_darray_free(objs); +} + +struct rb_gc_impl_each_objects_data { + int (*func)(void *, void *, size_t, void *); + void *data; +}; + +static int +rb_gc_impl_each_objects_i(VALUE obj, void *d) +{ + struct rb_gc_impl_each_objects_data *data = d; + + size_t slot_size = rb_gc_impl_obj_slot_size(obj); + + return data->func((void *)obj, (void *)(obj + slot_size), slot_size, data->data); +} + +void +rb_gc_impl_each_objects(void *objspace_ptr, int (*func)(void *, void *, size_t, void *), void *data) +{ + struct rb_gc_impl_each_objects_data each_objects_data = { + .func = func, + .data = data + }; + + each_object(objspace_ptr, rb_gc_impl_each_objects_i, &each_objects_data); +} + +struct rb_gc_impl_each_object_data { + void (*func)(VALUE, void *); + void *data; +}; + +static int +rb_gc_impl_each_object_i(VALUE obj, void *d) +{ + struct rb_gc_impl_each_object_data *data = d; + + data->func(obj, data->data); + + return 0; +} + +void +rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE, void *), void *data) +{ + struct rb_gc_impl_each_object_data each_object_data = { + .func = func, + .data = data + }; + + each_object(objspace_ptr, rb_gc_impl_each_object_i, &each_object_data); +} + +// Finalizers +static VALUE +gc_run_finalizers_get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i + 1); +} + +static void +gc_run_finalizers(void *data) +{ + struct objspace *objspace = data; + + rb_gc_set_pending_interrupt(); + + while (objspace->finalizer_jobs != NULL) { + struct MMTk_final_job *job = objspace->finalizer_jobs; + objspace->finalizer_jobs = job->next; + + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + job->as.dfree.func(job->as.dfree.data); + break; + case MMTK_FINAL_JOB_FINALIZE: { + VALUE finalizer_array = job->as.finalize.finalizer_array; + + rb_gc_run_obj_finalizer( + RARRAY_AREF(finalizer_array, 0), + RARRAY_LEN(finalizer_array) - 1, + gc_run_finalizers_get_final, + (void *)finalizer_array + ); + + RB_GC_GUARD(finalizer_array); + break; + } + } + + xfree(job); + } + + rb_gc_unset_pending_interrupt(); +} + +void +rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) +{ + if (dfree == NULL) return; + + struct objspace *objspace = objspace_ptr; + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->kind = MMTK_FINAL_JOB_DFREE; + job->as.dfree.func = dfree; + job->as.dfree.data = data; + + struct MMTk_final_job *prev; + do { + job->next = objspace->finalizer_jobs; + prev = RUBY_ATOMIC_PTR_CAS(objspace->finalizer_jobs, job->next, job); + } while (prev != job->next); + + if (!ruby_free_at_exit_p()) { + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } +} + +VALUE +rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) +{ + struct objspace *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + RBASIC(obj)->flags |= FL_FINALIZE; + + int lev = RB_GC_VM_LOCK(); + + if (st_lookup(objspace->finalizer_table, obj, &data)) { + table = (VALUE)data; + + /* avoid duplicate block, table is usually small */ + { + long len = RARRAY_LEN(table); + long i; + + for (i = 0; i < len; i++) { + VALUE recv = RARRAY_AREF(table, i); + if (rb_equal(recv, block)) { + RB_GC_VM_UNLOCK(lev); + return recv; + } + } + } + + rb_ary_push(table, block); + } + else { + table = rb_ary_new3(2, rb_obj_id(obj), block); + rb_obj_hide(table); + st_add_direct(objspace->finalizer_table, obj, table); + } + + RB_GC_VM_UNLOCK(lev); + + return block; +} + +void +rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) +{ + struct objspace *objspace = objspace_ptr; + + st_data_t data = obj; + + int lev = RB_GC_VM_LOCK(); + st_delete(objspace->finalizer_table, &data, 0); + RB_GC_VM_UNLOCK(lev); + + FL_UNSET(obj, FL_FINALIZE); +} + +void +rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) +{ + struct objspace *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + if (!FL_TEST(obj, FL_FINALIZE)) return; + + int lev = RB_GC_VM_LOCK(); + if (RB_LIKELY(st_lookup(objspace->finalizer_table, obj, &data))) { + table = rb_ary_dup((VALUE)data); + RARRAY_ASET(table, 0, rb_obj_id(dest)); + st_insert(objspace->finalizer_table, dest, table); + FL_SET(dest, FL_FINALIZE); + } + else { + rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj)); + } + RB_GC_VM_UNLOCK(lev); +} + +static int +move_finalizer_from_table_i(st_data_t key, st_data_t val, st_data_t arg) +{ + struct objspace *objspace = (struct objspace *)arg; + + make_final_job(objspace, (VALUE)key, (VALUE)val); + + return ST_DELETE; +} + +void +rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + while (objspace->finalizer_table->num_entries) { + st_foreach(objspace->finalizer_table, move_finalizer_from_table_i, (st_data_t)objspace); + + gc_run_finalizers(objspace); + } + + unsigned int lev = RB_GC_VM_LOCK(); + { + struct MMTk_ractor_cache *rc; + ccan_list_for_each(&objspace->ractor_caches, rc, list_node) { + mmtk_flush_obj_free_buffer(rc); + } + + struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates(); + for (size_t i = 0; i < registered_candidates.len; i++) { + VALUE obj = (VALUE)registered_candidates.ptr[i]; + + if (rb_gc_shutdown_call_finalizer_p(obj)) { + rb_gc_obj_free(objspace_ptr, obj); + RBASIC(obj)->flags = 0; + } + } + mmtk_free_raw_vec_of_obj_ref(registered_candidates); + } + RB_GC_VM_UNLOCK(lev); + + gc_run_finalizers(objspace); +} + +// Forking + +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + retry: + objspace->fork_hook_vm_lock_lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + /* At this point, we know that all the Ractors are paused because of the + * rb_gc_vm_barrier above. Since rb_mmtk_block_for_gc is a barrier point, + * one or more Ractors could be paused there. However, mmtk_before_fork is + * not compatible with that because it assumes that the MMTk workers are idle, + * but the workers are not idle because they are busy working on a GC. + * + * This essentially implements a trylock. It will optimistically lock but will + * release the lock if it detects that any other Ractors are waiting in + * rb_mmtk_block_for_gc. + */ + rb_atomic_t mutator_blocking_count = RUBY_ATOMIC_LOAD(objspace->mutator_blocking_count); + if (mutator_blocking_count != 0) { + RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev); + goto retry; + } + + mmtk_before_fork(); +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + struct objspace *objspace = objspace_ptr; + + mmtk_after_fork(rb_gc_get_ractor_newobj_cache()); + + RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev); +} + +// Statistics + +void +rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) +{ + struct objspace *objspace = objspace_ptr; + + objspace->measure_gc_time = RTEST(flag); +} + +bool +rb_gc_impl_get_measure_total_time(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->measure_gc_time; +} + +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->total_gc_time; +} + +size_t +rb_gc_impl_gc_count(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->gc_count; +} + +VALUE +rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE hash_or_key) +{ + VALUE hash = Qnil, key = Qnil; + + if (SYMBOL_P(hash_or_key)) { + key = hash_or_key; + } + else if (RB_TYPE_P(hash_or_key, T_HASH)) { + hash = hash_or_key; + } + else { + rb_bug("gc_info_decode: non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == ID2SYM(rb_intern_const(#name))) \ + return (attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, ID2SYM(rb_intern_const(#name)), (attr)); + + /* Hack to get StackProf working because it calls rb_gc_latest_gc_info with + * the :state key and expects a result. This always returns the :none state. */ + SET(state, ID2SYM(rb_intern_const("none"))); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +enum gc_stat_sym { + gc_stat_sym_count, + gc_stat_sym_moving_gc_count, + gc_stat_sym_time, + gc_stat_sym_total_allocated_objects, + gc_stat_sym_total_bytes, + gc_stat_sym_used_bytes, + gc_stat_sym_free_bytes, + gc_stat_sym_starting_heap_address, + gc_stat_sym_last_heap_address, + gc_stat_sym_weak_references_count, + gc_stat_sym_last +}; + +static VALUE gc_stat_symbols[gc_stat_sym_last]; + +static void +setup_gc_stat_symbols(void) +{ + if (gc_stat_symbols[0] == 0) { +#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s)) + S(count); + S(moving_gc_count); + S(time); + S(total_allocated_objects); + S(total_bytes); + S(used_bytes); + S(free_bytes); + S(starting_heap_address); + S(last_heap_address); + S(weak_references_count); + } +} + +VALUE +rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) +{ + struct objspace *objspace = objspace_ptr; + VALUE hash = Qnil, key = Qnil; + + setup_gc_stat_symbols(); + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + hash = hash_or_sym; + } + else if (SYMBOL_P(hash_or_sym)) { + key = hash_or_sym; + } + else { + rb_bug("non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == gc_stat_symbols[gc_stat_sym_##name]) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr)); + + SET(count, objspace->gc_count); + SET(moving_gc_count, objspace->moving_gc_count); + SET(time, objspace->total_gc_time / (1000 * 1000)); + SET(total_allocated_objects, objspace->total_allocated_objects); + SET(total_bytes, mmtk_total_bytes()); + SET(used_bytes, mmtk_used_bytes()); + SET(free_bytes, mmtk_free_bytes()); + SET(starting_heap_address, (size_t)mmtk_starting_heap_address()); + SET(last_heap_address, (size_t)mmtk_last_heap_address()); + SET(weak_references_count, mmtk_weak_references_count()); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +VALUE +rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) +{ + if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) { + int heap_idx = FIX2INT(heap_name); + if (heap_idx < 0 || heap_idx >= MMTK_HEAP_COUNT) { + rb_raise(rb_eArgError, "size pool index out of range"); + } + + if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) { + return SIZET2NUM(heap_sizes[heap_idx]); + } + + return Qundef; + } + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + return hash_or_sym; + } + + return Qundef; +} + +// Miscellaneous + +#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 1 +static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) +{ + static ID ID_object_id; + + if (!ID_object_id) { +#define I(s) ID_##s = rb_intern(#s); + I(object_id); +#undef I + } + + size_t n = 0; + +#define SET_ENTRY(na, v) do { \ + MMTK_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; + + return object_metadata_entries; +} + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + if (ptr == NULL) return false; + if ((uintptr_t)ptr % sizeof(void*) != 0) return false; + return mmtk_is_mmtk_object((MMTk_Address)ptr); +} + +bool +rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj) +{ + return false; +} + +void rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) { } + +void +rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) +{ + if (mmtk_object_wb_unprotected_p((MMTk_ObjectReference)obj)) { + rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest); + } + + rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj); +} + +// GC Identification + +const char * +rb_gc_impl_active_gc_name(void) +{ + return "mmtk"; +} diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h new file mode 100644 index 0000000000..b11e2873e3 --- /dev/null +++ b/gc/mmtk/mmtk.h @@ -0,0 +1,175 @@ +#ifndef MMTK_H +#define MMTK_H + +/* Warning, this file is autogenerated by cbindgen from the mmtk-ruby repository. Don't modify this manually. */ + +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +typedef struct MMTk_Builder MMTk_Builder; +typedef struct MMTk_Mutator MMTk_Mutator; + +typedef struct MMTk_ractor_cache *MMTk_VMThread; +typedef struct MMTk_ractor_cache *MMTk_VMMutatorThread; +typedef struct MMTk_GCThreadTLS *MMTk_VMWorkerThread; +typedef void *MMTk_Address; +typedef void *MMTk_ObjectReference; +typedef void *MMTk_NullableObjectReference; +typedef uint32_t MMTk_AllocationSemantics; + +typedef struct MMTk_BumpPointer { + uintptr_t cursor; + uintptr_t limit; +} MMTk_BumpPointer; + + +#define MMTk_OBJREF_OFFSET 8 + +#define MMTk_MIN_OBJ_ALIGN 8 + +#define MMTk_GC_THREAD_KIND_WORKER 1 + +typedef struct MMTk_RubyBindingOptions { + size_t suffix_size; +} MMTk_RubyBindingOptions; + +typedef MMTk_ObjectReference (*MMTk_ObjectClosureFunction)(void*, void*, MMTk_ObjectReference, bool); + +typedef struct MMTk_ObjectClosure { + /** + * The function to be called from C. + */ + MMTk_ObjectClosureFunction c_function; + /** + * The pointer to the Rust-level closure object. + */ + void *rust_closure; +} MMTk_ObjectClosure; + +typedef struct MMTk_GCThreadTLS { + int kind; + void *gc_context; + struct MMTk_ObjectClosure object_closure; +} MMTk_GCThreadTLS; + +typedef struct MMTk_RubyUpcalls { + void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls); + bool (*is_mutator)(void); + void (*stop_the_world)(void); + void (*resume_mutators)(bool gc_may_move); + void (*block_for_gc)(MMTk_VMMutatorThread tls); + void (*before_updating_jit_code)(void); + void (*after_updating_jit_code)(void); + size_t (*number_of_mutators)(void); + void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data); + void (*scan_gc_roots)(void); + void (*scan_objspace)(void); + void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to); + void (*update_object_references)(MMTk_ObjectReference object); + void (*call_gc_mark_children)(MMTk_ObjectReference object); + void (*handle_weak_references)(MMTk_ObjectReference object, bool moving); + void (*call_obj_free)(MMTk_ObjectReference object); + size_t (*vm_live_bytes)(void); + void (*update_global_tables)(int tbl_idx, bool moving); + int (*global_tables_count)(void); + void (*update_finalizer_table)(void); + bool (*special_const_p)(MMTk_ObjectReference object); + void (*mutator_thread_panic_handler)(void); + void (*gc_thread_panic_handler)(void); +} MMTk_RubyUpcalls; + +typedef struct MMTk_RawVecOfObjRef { + MMTk_ObjectReference *ptr; + size_t len; + size_t capa; +} MMTk_RawVecOfObjRef; + +bool mmtk_is_live_object(MMTk_ObjectReference object); + +bool mmtk_is_reachable(MMTk_ObjectReference object); + +MMTk_Builder *mmtk_builder_default(void); + +void mmtk_init_binding(MMTk_Builder *builder, + const struct MMTk_RubyBindingOptions *binding_options, + const struct MMTk_RubyUpcalls *upcalls); + +void mmtk_initialize_collection(MMTk_VMThread tls); + +MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls); + +MMTk_BumpPointer *mmtk_get_bump_pointer_allocator(MMTk_Mutator *m); + +void mmtk_destroy_mutator(MMTk_Mutator *mutator); + +void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls, bool force, bool exhaustive); + +void mmtk_set_gc_enabled(bool enable); + +bool mmtk_gc_enabled_p(void); + +MMTk_Address mmtk_alloc(MMTk_Mutator *mutator, + size_t size, + size_t align, + size_t offset, + MMTk_AllocationSemantics semantics); + +void mmtk_post_alloc(MMTk_Mutator *mutator, + MMTk_ObjectReference refer, + size_t bytes, + MMTk_AllocationSemantics semantics); + +void mmtk_add_obj_free_candidates(const MMTk_ObjectReference *objects, + size_t count, + bool can_parallel_free); + +void mmtk_declare_weak_references(MMTk_ObjectReference object); + +bool mmtk_weak_references_alive_p(MMTk_ObjectReference object); + +size_t mmtk_weak_references_count(void); + +void mmtk_register_pinning_obj(MMTk_ObjectReference obj); + +void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object); + +void mmtk_register_wb_unprotected_object(MMTk_ObjectReference object); + +bool mmtk_object_wb_unprotected_p(MMTk_ObjectReference object); + +void mmtk_enumerate_objects(void (*callback)(MMTk_ObjectReference, void*), void *data); + +struct MMTk_RawVecOfObjRef mmtk_get_all_obj_free_candidates(void); + +void mmtk_free_raw_vec_of_obj_ref(struct MMTk_RawVecOfObjRef raw_vec); + +void mmtk_before_fork(void); + +void mmtk_after_fork(MMTk_VMThread tls); + +size_t mmtk_total_bytes(void); + +size_t mmtk_used_bytes(void); + +size_t mmtk_free_bytes(void); + +MMTk_Address mmtk_starting_heap_address(void); + +MMTk_Address mmtk_last_heap_address(void); + +size_t mmtk_worker_count(void); + +const uint8_t *mmtk_plan(void); + +const uint8_t *mmtk_heap_mode(void); + +size_t mmtk_heap_min(void); + +size_t mmtk_heap_max(void); + +bool mmtk_is_mmtk_object(MMTk_Address addr); + +#endif /* MMTK_H */ diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs new file mode 100644 index 0000000000..30890e0853 --- /dev/null +++ b/gc/mmtk/src/abi.rs @@ -0,0 +1,335 @@ +use crate::api::RubyMutator; +use crate::extra_assert; +use crate::Ruby; +use libc::c_int; +use mmtk::scheduler::GCWorker; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMWorkerThread; + +// For the C binding +pub const OBJREF_OFFSET: usize = 8; +pub const MIN_OBJ_ALIGN: usize = 8; // Even on 32-bit machine. A Ruby object is at least 40 bytes large. + +pub const GC_THREAD_KIND_WORKER: libc::c_int = 1; + +const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; + +// An opaque type for the C counterpart. +#[allow(non_camel_case_types)] +pub struct st_table; + +#[repr(C)] +pub struct HiddenHeader { + pub prefix: usize, +} + +impl HiddenHeader { + #[inline(always)] + pub fn is_sane(&self) -> bool { + self.prefix & !HIDDEN_SIZE_MASK == 0 + } + + #[inline(always)] + fn assert_sane(&self) { + extra_assert!( + self.is_sane(), + "Hidden header is corrupted: {:x}", + self.prefix + ); + } + + pub fn payload_size(&self) -> usize { + self.assert_sane(); + self.prefix & HIDDEN_SIZE_MASK + } +} + +/// Provide convenient methods for accessing Ruby objects. +/// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. +pub struct RubyObjectAccess { + objref: ObjectReference, +} + +impl RubyObjectAccess { + pub fn from_objref(objref: ObjectReference) -> Self { + Self { objref } + } + + pub fn obj_start(&self) -> Address { + self.objref.to_raw_address().sub(Self::prefix_size()) + } + + pub fn payload_addr(&self) -> Address { + self.objref.to_raw_address() + } + + pub fn suffix_addr(&self) -> Address { + self.objref.to_raw_address().add(self.payload_size()) + } + + pub fn obj_end(&self) -> Address { + self.suffix_addr() + Self::suffix_size() + } + + fn hidden_header(&self) -> &'static HiddenHeader { + unsafe { self.obj_start().as_ref() } + } + + #[allow(unused)] // Maybe we need to mutate the hidden header in the future. + fn hidden_header_mut(&self) -> &'static mut HiddenHeader { + unsafe { self.obj_start().as_mut_ref() } + } + + pub fn payload_size(&self) -> usize { + self.hidden_header().payload_size() + } + + fn flags_field(&self) -> Address { + self.objref.to_raw_address() + } + + pub fn load_flags(&self) -> usize { + unsafe { self.flags_field().load::<usize>() } + } + + pub fn prefix_size() -> usize { + // Currently, a hidden size field of word size is placed before each object. + OBJREF_OFFSET + } + + pub fn suffix_size() -> usize { + // In RACTOR_CHECK_MODE, Ruby hides a field after each object to hold the Ractor ID. + unsafe { crate::BINDING_FAST.suffix_size } + } + + pub fn object_size(&self) -> usize { + Self::prefix_size() + self.payload_size() + Self::suffix_size() + } +} + +type ObjectClosureFunction = + extern "C" fn(*mut libc::c_void, *mut libc::c_void, ObjectReference, bool) -> ObjectReference; + +#[repr(C)] +pub struct ObjectClosure { + /// The function to be called from C. + pub c_function: ObjectClosureFunction, + /// The pointer to the Rust-level closure object. + pub rust_closure: *mut libc::c_void, +} + +impl Default for ObjectClosure { + fn default() -> Self { + Self { + c_function: THE_UNREGISTERED_CLOSURE_FUNC, + rust_closure: std::ptr::null_mut(), + } + } +} + +/// Rust doesn't require function items to have a unique address. +/// We therefore force using this particular constant. +/// +/// See: https://rust-lang.github.io/rust-clippy/master/index.html#fn_address_comparisons +const THE_UNREGISTERED_CLOSURE_FUNC: ObjectClosureFunction = ObjectClosure::c_function_unregistered; + +impl ObjectClosure { + /// Set this ObjectClosure temporarily to `visit_object`, and execute `f`. During the execution of + /// `f`, the Ruby VM may call this ObjectClosure. When the Ruby VM calls this ObjectClosure, + /// it effectively calls `visit_object`. + /// + /// This method is intended to run Ruby VM code in `f` with temporarily modified behavior of + /// `rb_gc_mark`, `rb_gc_mark_movable` and `rb_gc_location` + /// + /// Both `f` and `visit_object` may access and modify local variables in the environment where + /// `set_temporarily_and_run_code` called. + /// + /// Note that this function is not reentrant. Don't call this function in either `callback` or + /// `f`. + pub fn set_temporarily_and_run_code<'env, T, F1, F2>( + &mut self, + mut visit_object: F1, + f: F2, + ) -> T + where + F1: 'env + FnMut(&'static mut GCWorker<Ruby>, ObjectReference, bool) -> ObjectReference, + F2: 'env + FnOnce() -> T, + { + debug_assert!( + std::ptr::fn_addr_eq(self.c_function, THE_UNREGISTERED_CLOSURE_FUNC), + "set_temporarily_and_run_code is recursively called." + ); + self.c_function = Self::c_function_registered::<F1>; + self.rust_closure = &mut visit_object as *mut F1 as *mut libc::c_void; + let result = f(); + *self = Default::default(); + result + } + + extern "C" fn c_function_registered<F>( + rust_closure: *mut libc::c_void, + worker: *mut libc::c_void, + object: ObjectReference, + pin: bool, + ) -> ObjectReference + where + F: FnMut(&'static mut GCWorker<Ruby>, ObjectReference, bool) -> ObjectReference, + { + let rust_closure = unsafe { &mut *(rust_closure as *mut F) }; + let worker = unsafe { &mut *(worker as *mut GCWorker<Ruby>) }; + rust_closure(worker, object, pin) + } + + extern "C" fn c_function_unregistered( + _rust_closure: *mut libc::c_void, + worker: *mut libc::c_void, + object: ObjectReference, + pin: bool, + ) -> ObjectReference { + let worker = unsafe { &mut *(worker as *mut GCWorker<Ruby>) }; + panic!( + "object_closure is not set. worker ordinal: {}, object: {}, pin: {}", + worker.ordinal, object, pin + ); + } +} + +#[repr(C)] +pub struct GCThreadTLS { + pub kind: libc::c_int, + pub gc_context: *mut libc::c_void, + pub object_closure: ObjectClosure, +} + +impl GCThreadTLS { + fn new(kind: libc::c_int, gc_context: *mut libc::c_void) -> Self { + Self { + kind, + gc_context, + object_closure: Default::default(), + } + } + + pub fn for_worker(gc_context: *mut GCWorker<Ruby>) -> Self { + Self::new(GC_THREAD_KIND_WORKER, gc_context as *mut libc::c_void) + } + + pub fn from_vwt(vwt: VMWorkerThread) -> *mut GCThreadTLS { + unsafe { std::mem::transmute(vwt) } + } + + /// Cast a pointer to `GCThreadTLS` to a ref, with assertion for null pointer. + /// + /// # Safety + /// + /// Has undefined behavior if `ptr` is invalid. + pub unsafe fn check_cast(ptr: *mut GCThreadTLS) -> &'static mut GCThreadTLS { + assert!(!ptr.is_null()); + let result = unsafe { &mut *ptr }; + debug_assert!({ + let kind = result.kind; + kind == GC_THREAD_KIND_WORKER + }); + result + } + + /// Cast a pointer to `VMWorkerThread` to a ref, with assertion for null pointer. + /// + /// # Safety + /// + /// Has undefined behavior if `ptr` is invalid. + pub unsafe fn from_vwt_check(vwt: VMWorkerThread) -> &'static mut GCThreadTLS { + let ptr = Self::from_vwt(vwt); + unsafe { Self::check_cast(ptr) } + } + + #[allow(clippy::not_unsafe_ptr_arg_deref)] // `transmute` does not dereference pointer + pub fn to_vwt(ptr: *mut Self) -> VMWorkerThread { + unsafe { std::mem::transmute(ptr) } + } + + pub fn worker<'w>(&mut self) -> &'w mut GCWorker<Ruby> { + // NOTE: The returned ref points to the worker which does not have the same lifetime as self. + assert!(self.kind == GC_THREAD_KIND_WORKER); + unsafe { &mut *(self.gc_context as *mut GCWorker<Ruby>) } + } +} + +#[repr(C)] +#[derive(Clone)] +pub struct RawVecOfObjRef { + pub ptr: *mut ObjectReference, + pub len: usize, + pub capa: usize, +} + +impl RawVecOfObjRef { + pub fn from_vec(vec: Vec<ObjectReference>) -> RawVecOfObjRef { + // Note: Vec::into_raw_parts is unstable. We implement it manually. + let mut vec = std::mem::ManuallyDrop::new(vec); + let (ptr, len, capa) = (vec.as_mut_ptr(), vec.len(), vec.capacity()); + + RawVecOfObjRef { ptr, len, capa } + } + + /// # Safety + /// + /// This function turns raw pointer into a Vec without check. + pub unsafe fn into_vec(self) -> Vec<ObjectReference> { + unsafe { Vec::from_raw_parts(self.ptr, self.len, self.capa) } + } +} + +impl From<Vec<ObjectReference>> for RawVecOfObjRef { + fn from(v: Vec<ObjectReference>) -> Self { + Self::from_vec(v) + } +} + +#[repr(C)] +#[derive(Clone)] +pub struct RubyBindingOptions { + pub suffix_size: usize, +} + +#[repr(C)] +#[derive(Clone)] +pub struct RubyUpcalls { + pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS), + pub is_mutator: extern "C" fn() -> bool, + pub stop_the_world: extern "C" fn(), + pub resume_mutators: extern "C" fn(gc_may_move: bool), + pub block_for_gc: extern "C" fn(tls: VMMutatorThread), + pub before_updating_jit_code: extern "C" fn(), + pub after_updating_jit_code: extern "C" fn(), + pub number_of_mutators: extern "C" fn() -> usize, + pub get_mutators: extern "C" fn( + visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void), + data: *mut libc::c_void, + ), + pub scan_gc_roots: extern "C" fn(), + pub scan_objspace: extern "C" fn(), + pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference), + pub update_object_references: extern "C" fn(object: ObjectReference), + pub call_gc_mark_children: extern "C" fn(object: ObjectReference), + pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool), + pub call_obj_free: extern "C" fn(object: ObjectReference), + pub vm_live_bytes: extern "C" fn() -> usize, + pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool), + pub global_tables_count: extern "C" fn() -> c_int, + pub update_finalizer_table: extern "C" fn(), + pub special_const_p: extern "C" fn(object: ObjectReference) -> bool, + pub mutator_thread_panic_handler: extern "C" fn(), + pub gc_thread_panic_handler: extern "C" fn(), +} + +unsafe impl Sync for RubyUpcalls {} + +#[repr(C)] +#[derive(Clone)] +pub struct HeapBounds { + pub start: *mut libc::c_void, + pub end: *mut libc::c_void, +} diff --git a/gc/mmtk/src/active_plan.rs b/gc/mmtk/src/active_plan.rs new file mode 100644 index 0000000000..80372a7576 --- /dev/null +++ b/gc/mmtk/src/active_plan.rs @@ -0,0 +1,56 @@ +use std::collections::VecDeque; +use std::marker::PhantomData; + +use crate::mmtk; +use crate::upcalls; +use crate::Ruby; +use mmtk::util::opaque_pointer::*; +use mmtk::vm::ActivePlan; +use mmtk::Mutator; + +pub struct VMActivePlan {} + +impl ActivePlan<Ruby> for VMActivePlan { + fn number_of_mutators() -> usize { + (upcalls().number_of_mutators)() + } + + fn is_mutator(_tls: VMThread) -> bool { + (upcalls().is_mutator)() + } + + fn mutator(_tls: VMMutatorThread) -> &'static mut Mutator<Ruby> { + unimplemented!() + } + + fn mutators<'a>() -> Box<dyn Iterator<Item = &'a mut Mutator<Ruby>> + 'a> { + let mut mutators = VecDeque::new(); + (upcalls().get_mutators)( + add_mutator_to_vec, + &mut mutators as *mut VecDeque<&mut Mutator<Ruby>> as _, + ); + + Box::new(RubyMutatorIterator { + mutators, + phantom_data: PhantomData, + }) + } +} + +extern "C" fn add_mutator_to_vec(mutator: *mut Mutator<Ruby>, mutators: *mut libc::c_void) { + let mutators = unsafe { &mut *(mutators as *mut VecDeque<*mut Mutator<Ruby>>) }; + mutators.push_back(unsafe { &mut *mutator }); +} + +struct RubyMutatorIterator<'a> { + mutators: VecDeque<&'a mut Mutator<Ruby>>, + phantom_data: PhantomData<&'a ()>, +} + +impl<'a> Iterator for RubyMutatorIterator<'a> { + type Item = &'a mut Mutator<Ruby>; + + fn next(&mut self) -> Option<Self::Item> { + self.mutators.pop_front() + } +} diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs new file mode 100644 index 0000000000..c0540fe0c8 --- /dev/null +++ b/gc/mmtk/src/api.rs @@ -0,0 +1,551 @@ +// Functions in this module are unsafe for one reason: +// They are called by C functions and they need to pass raw pointers to Rust. +#![allow(clippy::missing_safety_doc)] + +use mmtk::util::alloc::BumpPointer; +use mmtk::util::alloc::ImmixAllocator; +use mmtk::util::conversions; +use mmtk::util::options::PlanSelector; +use std::str::FromStr; +use std::sync::atomic::Ordering; + +use crate::abi::RawVecOfObjRef; +use crate::abi::RubyBindingOptions; +use crate::abi::RubyUpcalls; +use crate::binding; +use crate::binding::RubyBinding; +use crate::heap::CpuHeapTriggerConfig; +use crate::heap::RubyHeapTriggerConfig; +use crate::heap::CPU_HEAP_TRIGGER_CONFIG; +use crate::heap::RUBY_HEAP_TRIGGER_CONFIG; +use crate::mmtk; +use crate::utils::default_heap_max; +use crate::utils::parse_capacity; +use crate::Ruby; +use crate::RubySlot; +use mmtk::memory_manager; +use mmtk::memory_manager::mmtk_init; +use mmtk::util::constants::MIN_OBJECT_SIZE; +use mmtk::util::options::GCTriggerSelector; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; +use mmtk::AllocationSemantics; +use mmtk::MMTKBuilder; +use mmtk::Mutator; + +pub type RubyMutator = Mutator<Ruby>; + +#[no_mangle] +pub extern "C" fn mmtk_is_live_object(object: ObjectReference) -> bool { + memory_manager::is_live_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_is_reachable(object: ObjectReference) -> bool { + object.is_reachable() +} + +// =============== Bootup =============== + +fn parse_env_var_with<T, F: FnOnce(&str) -> Option<T>>(key: &str, parse: F) -> Option<T> { + let val = match std::env::var(key) { + Ok(val) => val, + Err(std::env::VarError::NotPresent) => return None, + Err(std::env::VarError::NotUnicode(os_string)) => { + eprintln!("[FATAL] Invalid {key} {os_string:?}"); + std::process::exit(1); + } + }; + + let parsed = parse(&val).unwrap_or_else(|| { + eprintln!("[FATAL] Invalid {key} {val}"); + std::process::exit(1); + }); + + Some(parsed) +} + +fn parse_env_var<T: FromStr>(key: &str) -> Option<T> { + parse_env_var_with(key, |s| s.parse().ok()) +} + +fn mmtk_builder_default_parse_threads() -> Option<usize> { + parse_env_var("MMTK_THREADS") +} + +fn mmtk_builder_default_parse_heap_min() -> usize { + const DEFAULT_HEAP_MIN: usize = 1 << 20; + parse_env_var_with("MMTK_HEAP_MIN", parse_capacity).unwrap_or(DEFAULT_HEAP_MIN) +} + +fn mmtk_builder_default_parse_heap_max() -> usize { + parse_env_var_with("MMTK_HEAP_MAX", parse_capacity).unwrap_or_else(default_heap_max) +} + +fn parse_float_env_var(key: &str, default: f64, min: f64, max: f64) -> f64 { + parse_env_var_with(key, |s| { + let mut float = f64::from_str(s).unwrap_or(default); + + if float <= min { + eprintln!( + "{key} has value {float} which must be greater than {min}, using default instead" + ); + float = default; + } + + if float >= max { + eprintln!( + "{key} has value {float} which must be less than {max}, using default instead" + ); + float = default; + } + + Some(float) + }) + .unwrap_or(default) +} + +fn mmtk_builder_default_parse_heap_mode(heap_min: usize, heap_max: usize) -> GCTriggerSelector { + let make_fixed = || GCTriggerSelector::FixedHeapSize(heap_max); + let make_dynamic = || GCTriggerSelector::DynamicHeapSize(heap_min, heap_max); + + parse_env_var_with("MMTK_HEAP_MODE", |s| match s { + "fixed" => Some(make_fixed()), + "dynamic" => Some(make_dynamic()), + "ruby" => { + let min_ratio = parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", 0.2, 0.0, 1.0); + let goal_ratio = + parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", 0.4, min_ratio, 1.0); + let max_ratio = + parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", 0.65, goal_ratio, 1.0); + + crate::heap::RUBY_HEAP_TRIGGER_CONFIG + .set(RubyHeapTriggerConfig { + min_heap_pages: conversions::bytes_to_pages_up(heap_min), + max_heap_pages: conversions::bytes_to_pages_up(heap_max), + heap_pages_min_ratio: min_ratio, + heap_pages_goal_ratio: goal_ratio, + heap_pages_max_ratio: max_ratio, + }) + .unwrap_or_else(|_| panic!("RUBY_HEAP_TRIGGER_CONFIG is already set")); + + Some(GCTriggerSelector::Delegated) + } + "cpu" => { + // CPU-overhead-driven heap sizing based on Tavakolisomeh et al., + // "Heap Size Adjustment with CPU Control", MPLR '23. + // + // Target is expressed as a percentage (0, 100) via + // `MMTK_GC_CPU_TARGET`. The paper recommends 15 for ZGC (a + // concurrent collector); we default to 5 for MMTk-Ruby. With + // MMTk's stop-the-world Immix, every percent of GC CPU is also + // a percent of wall-clock the mutator is blocked on, so a much + // smaller budget is appropriate. An empirical sweep across + // ruby-bench (railsbench, lobsters, psych-load, liquid-render, + // lee) found target=5 to be Pareto-optimal: ~6% geomean speedup + // vs. the `ruby` heap mode with effectively identical geomean + // peak RSS. + let target_percent = parse_float_env_var("MMTK_GC_CPU_TARGET", 5.0, 0.0, 100.0); + let window_size = parse_env_var::<usize>("MMTK_GC_CPU_WINDOW").unwrap_or(3); + let window_size = window_size.max(1); + + let min_heap_pages = conversions::bytes_to_pages_up(heap_min); + let max_heap_pages = conversions::bytes_to_pages_up(heap_max); + // Start at the min heap size, as the other delegated triggers do. + // The control loop will adjust from here after the first GC cycle. + let initial_heap_pages = min_heap_pages; + + CPU_HEAP_TRIGGER_CONFIG + .set(CpuHeapTriggerConfig { + min_heap_pages, + max_heap_pages, + initial_heap_pages, + target_gc_cpu: target_percent / 100.0, + window_size, + }) + .unwrap_or_else(|_| panic!("CPU_HEAP_TRIGGER_CONFIG is already set")); + + Some(GCTriggerSelector::Delegated) + } + _ => None, + }) + .unwrap_or_else(make_dynamic) +} + +fn mmtk_builder_default_parse_plan() -> PlanSelector { + parse_env_var_with("MMTK_PLAN", |s| match s { + "NoGC" => Some(PlanSelector::NoGC), + "MarkSweep" => Some(PlanSelector::MarkSweep), + "Immix" => Some(PlanSelector::Immix), + _ => None, + }) + .unwrap_or(PlanSelector::Immix) +} + +#[no_mangle] +pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder { + let mut builder = MMTKBuilder::new_no_env_vars(); + builder.options.no_finalizer.set(true); + + if let Some(threads) = mmtk_builder_default_parse_threads() { + if !builder.options.threads.set(threads) { + // MMTk will validate it and reject 0. + eprintln!("[FATAL] Failed to set the number of MMTk threads to {threads}"); + std::process::exit(1); + } + } + + let heap_min = mmtk_builder_default_parse_heap_min(); + + let heap_max = mmtk_builder_default_parse_heap_max(); + + if heap_min >= heap_max { + eprintln!("[FATAL] MMTK_HEAP_MIN({heap_min}) >= MMTK_HEAP_MAX({heap_max})"); + std::process::exit(1); + } + + builder + .options + .gc_trigger + .set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max)); + + builder.options.plan.set(mmtk_builder_default_parse_plan()); + + Box::into_raw(Box::new(builder)) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_init_binding( + builder: *mut MMTKBuilder, + binding_options: *const RubyBindingOptions, + upcalls: *const RubyUpcalls, +) { + crate::MUTATOR_THREAD_PANIC_HANDLER + .set((unsafe { (*upcalls).clone() }).mutator_thread_panic_handler) + .unwrap_or_else(|_| panic!("MUTATOR_THREAD_PANIC_HANDLER is already initialized")); + + crate::set_panic_hook(); + + let builder: Box<MMTKBuilder> = unsafe { Box::from_raw(builder) }; + let binding_options = unsafe { (*binding_options).clone() }; + let mmtk_boxed = mmtk_init(&builder); + let mmtk_static = Box::leak(Box::new(mmtk_boxed)); + + let mut binding = RubyBinding::new(mmtk_static, &binding_options, upcalls); + binding + .weak_proc + .init_parallel_obj_free_candidates(memory_manager::num_of_workers(binding.mmtk)); + + crate::BINDING + .set(binding) + .unwrap_or_else(|_| panic!("Binding is already initialized")); +} + +#[no_mangle] +pub extern "C" fn mmtk_initialize_collection(tls: VMThread) { + memory_manager::initialize_collection(mmtk(), tls) +} + +#[no_mangle] +pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator { + Box::into_raw(memory_manager::bind_mutator(mmtk(), tls)) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_get_bump_pointer_allocator(m: *mut RubyMutator) -> *mut BumpPointer { + match *crate::BINDING.get().unwrap().mmtk.get_options().plan { + PlanSelector::Immix => { + let mutator: &mut Mutator<Ruby> = unsafe { &mut *m }; + let allocator = + unsafe { mutator.allocator_mut(mmtk::util::alloc::AllocatorSelector::Immix(0)) }; + + if let Some(immix_allocator) = allocator.downcast_mut::<ImmixAllocator<Ruby>>() { + &mut immix_allocator.bump_pointer as *mut BumpPointer + } else { + panic!("Failed to get bump pointer allocator"); + } + } + _ => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) { + // notify mmtk-core about destroyed mutator + memory_manager::destroy_mutator(unsafe { &mut *mutator }); + // turn the ptr back to a box, and let Rust properly reclaim it + let _ = unsafe { Box::from_raw(mutator) }; +} + +// =============== GC =============== + +#[no_mangle] +pub extern "C" fn mmtk_handle_user_collection_request( + tls: VMMutatorThread, + force: bool, + exhaustive: bool, +) { + crate::mmtk().handle_user_collection_request(tls, force, exhaustive); +} + +#[no_mangle] +pub extern "C" fn mmtk_set_gc_enabled(enable: bool) { + crate::CONFIGURATION + .gc_enabled + .store(enable, Ordering::Relaxed); +} + +#[no_mangle] +pub extern "C" fn mmtk_gc_enabled_p() -> bool { + crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed) +} + +// =============== Object allocation =============== + +#[no_mangle] +pub unsafe extern "C" fn mmtk_alloc( + mutator: *mut RubyMutator, + size: usize, + align: usize, + offset: usize, + semantics: AllocationSemantics, +) -> Address { + let clamped_size = size.max(MIN_OBJECT_SIZE); + memory_manager::alloc::<Ruby>( + unsafe { &mut *mutator }, + clamped_size, + align, + offset, + semantics, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_post_alloc( + mutator: *mut RubyMutator, + refer: ObjectReference, + bytes: usize, + semantics: AllocationSemantics, +) { + memory_manager::post_alloc::<Ruby>(unsafe { &mut *mutator }, refer, bytes, semantics) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_add_obj_free_candidates( + objects: *const ObjectReference, + count: usize, + can_parallel_free: bool, +) { + let objects = unsafe { std::slice::from_raw_parts(objects, count) }; + binding() + .weak_proc + .add_obj_free_candidates_batch(objects, can_parallel_free) +} + +// =============== Weak references =============== + +#[no_mangle] +pub extern "C" fn mmtk_declare_weak_references(object: ObjectReference) { + binding().weak_proc.add_weak_reference(object); +} + +#[no_mangle] +pub extern "C" fn mmtk_weak_references_alive_p(object: ObjectReference) -> bool { + object.is_reachable() +} + +#[no_mangle] +pub extern "C" fn mmtk_weak_references_count() -> usize { + binding().weak_proc.weak_references_count() +} + +// =============== Compaction =============== + +#[no_mangle] +pub extern "C" fn mmtk_register_pinning_obj(obj: ObjectReference) { + crate::binding().pinning_registry.register(obj); +} + +// =============== Write barriers =============== + +#[no_mangle] +pub unsafe extern "C" fn mmtk_object_reference_write_post( + mutator: *mut RubyMutator, + object: ObjectReference, +) { + let ignored_slot = RubySlot::from_address(Address::ZERO); + let ignored_target = ObjectReference::from_raw_address(Address::ZERO); + mmtk::memory_manager::object_reference_write_post( + unsafe { &mut *mutator }, + object, + ignored_slot, + ignored_target, + ) +} + +#[no_mangle] +pub extern "C" fn mmtk_register_wb_unprotected_object(object: ObjectReference) { + crate::binding().register_wb_unprotected_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_object_wb_unprotected_p(object: ObjectReference) -> bool { + crate::binding().object_wb_unprotected_p(object) +} + +// =============== Heap walking =============== + +#[no_mangle] +pub extern "C" fn mmtk_enumerate_objects( + callback: extern "C" fn(ObjectReference, *mut libc::c_void), + data: *mut libc::c_void, +) { + crate::mmtk().enumerate_objects(|object| { + callback(object, data); + }) +} + +// =============== Finalizers =============== + +#[no_mangle] +pub extern "C" fn mmtk_get_all_obj_free_candidates() -> RawVecOfObjRef { + let vec = binding().weak_proc.get_all_obj_free_candidates(); + RawVecOfObjRef::from_vec(vec) +} + +#[no_mangle] +pub extern "C" fn mmtk_free_raw_vec_of_obj_ref(raw_vec: RawVecOfObjRef) { + unsafe { raw_vec.into_vec() }; +} + +// =============== Forking =============== + +#[no_mangle] +pub extern "C" fn mmtk_before_fork() { + mmtk().prepare_to_fork(); + binding().join_all_gc_threads(); +} + +#[no_mangle] +pub extern "C" fn mmtk_after_fork(tls: VMThread) { + mmtk().after_fork(tls); +} + +// =============== Statistics =============== + +#[no_mangle] +pub extern "C" fn mmtk_total_bytes() -> usize { + memory_manager::total_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_used_bytes() -> usize { + memory_manager::used_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_free_bytes() -> usize { + memory_manager::free_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_starting_heap_address() -> Address { + memory_manager::starting_heap_address() +} + +#[no_mangle] +pub extern "C" fn mmtk_last_heap_address() -> Address { + memory_manager::last_heap_address() +} + +#[no_mangle] +pub extern "C" fn mmtk_worker_count() -> usize { + memory_manager::num_of_workers(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_plan() -> *const u8 { + static NO_GC: &[u8] = b"NoGC\0"; + static MARK_SWEEP: &[u8] = b"MarkSweep\0"; + static IMMIX: &[u8] = b"Immix\0"; + + match *crate::BINDING.get().unwrap().mmtk.get_options().plan { + PlanSelector::NoGC => NO_GC.as_ptr(), + PlanSelector::MarkSweep => MARK_SWEEP.as_ptr(), + PlanSelector::Immix => IMMIX.as_ptr(), + _ => panic!("Unknown plan"), + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_mode() -> *const u8 { + static FIXED_HEAP: &[u8] = b"fixed\0"; + static DYNAMIC_HEAP: &[u8] = b"dynamic\0"; + static RUBY_HEAP: &[u8] = b"ruby\0"; + static CPU_HEAP: &[u8] = b"cpu\0"; + + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(_) => FIXED_HEAP.as_ptr(), + GCTriggerSelector::DynamicHeapSize(_, _) => DYNAMIC_HEAP.as_ptr(), + GCTriggerSelector::Delegated => { + // Two delegated triggers exist; disambiguate via the populated + // config singleton. + if CPU_HEAP_TRIGGER_CONFIG.get().is_some() { + CPU_HEAP.as_ptr() + } else { + RUBY_HEAP.as_ptr() + } + } + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_min() -> usize { + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(_) => 0, + GCTriggerSelector::DynamicHeapSize(min_size, _) => min_size, + GCTriggerSelector::Delegated => { + if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() { + conversions::pages_to_bytes(cfg.min_heap_pages) + } else { + conversions::pages_to_bytes( + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("RUBY_HEAP_TRIGGER_CONFIG not set") + .min_heap_pages, + ) + } + } + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_max() -> usize { + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(max_size) => max_size, + GCTriggerSelector::DynamicHeapSize(_, max_size) => max_size, + GCTriggerSelector::Delegated => { + if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() { + conversions::pages_to_bytes(cfg.max_heap_pages) + } else { + conversions::pages_to_bytes( + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("RUBY_HEAP_TRIGGER_CONFIG not set") + .max_heap_pages, + ) + } + } + } +} + +// =============== Miscellaneous =============== + +#[no_mangle] +pub extern "C" fn mmtk_is_mmtk_object(addr: Address) -> bool { + debug_assert!(!addr.is_zero()); + debug_assert!(addr.is_aligned_to(mmtk::util::is_mmtk_object::VO_BIT_REGION_SIZE)); + memory_manager::is_mmtk_object(addr).is_some() +} diff --git a/gc/mmtk/src/binding.rs b/gc/mmtk/src/binding.rs new file mode 100644 index 0000000000..36d4a992fd --- /dev/null +++ b/gc/mmtk/src/binding.rs @@ -0,0 +1,129 @@ +use std::collections::HashSet; +use std::ffi::CString; +use std::sync::atomic::AtomicBool; +use std::sync::Mutex; +use std::thread::JoinHandle; + +use mmtk::util::ObjectReference; +use mmtk::MMTK; + +use crate::abi; +use crate::abi::RubyBindingOptions; +use crate::pinning_registry::PinningRegistry; +use crate::weak_proc::WeakProcessor; +use crate::Ruby; + +pub struct RubyBindingFast { + pub suffix_size: usize, +} + +impl Default for RubyBindingFast { + fn default() -> Self { + Self::new() + } +} + +impl RubyBindingFast { + pub const fn new() -> Self { + Self { suffix_size: 0 } + } +} + +pub struct RubyConfiguration { + pub gc_enabled: AtomicBool, +} + +impl Default for RubyConfiguration { + fn default() -> Self { + Self::new() + } +} + +impl RubyConfiguration { + pub const fn new() -> Self { + Self { + // Mimic the old behavior when the gc_enabled flag was in mmtk-core. + // We may refactor it so that it is false by default. + gc_enabled: AtomicBool::new(true), + } + } +} + +pub struct RubyBinding { + pub mmtk: &'static MMTK<Ruby>, + pub options: RubyBindingOptions, + pub upcalls: *const abi::RubyUpcalls, + pub plan_name: Mutex<Option<CString>>, + pub weak_proc: WeakProcessor, + pub pinning_registry: PinningRegistry, + pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>, + pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>, +} + +unsafe impl Sync for RubyBinding {} +unsafe impl Send for RubyBinding {} + +impl RubyBinding { + pub fn new( + mmtk: &'static MMTK<Ruby>, + binding_options: &RubyBindingOptions, + upcalls: *const abi::RubyUpcalls, + ) -> Self { + unsafe { + crate::BINDING_FAST.suffix_size = binding_options.suffix_size; + } + + Self { + mmtk, + options: binding_options.clone(), + upcalls, + plan_name: Mutex::new(None), + weak_proc: WeakProcessor::new(), + pinning_registry: PinningRegistry::new(), + gc_thread_join_handles: Default::default(), + wb_unprotected_objects: Default::default(), + } + } + + pub fn upcalls(&self) -> &'static abi::RubyUpcalls { + unsafe { &*self.upcalls as &'static abi::RubyUpcalls } + } + + pub fn get_plan_name_c(&self) -> *const libc::c_char { + let mut plan_name = self.plan_name.lock().unwrap(); + if plan_name.is_none() { + let name_string = format!("{:?}", *self.mmtk.get_options().plan); + let c_string = CString::new(name_string) + .unwrap_or_else(|e| panic!("Failed converting plan name to CString: {e}")); + *plan_name = Some(c_string); + } + plan_name.as_deref().unwrap().as_ptr() + } + + pub fn join_all_gc_threads(&self) { + let handles = { + let mut guard = self.gc_thread_join_handles.lock().unwrap(); + std::mem::take(&mut *guard) + }; + + debug!("Joining GC threads..."); + let total = handles.len(); + let mut joined = 0; + for handle in handles { + handle.join().unwrap(); + joined += 1; + debug!("{joined}/{total} GC threads joined."); + } + } + + pub fn register_wb_unprotected_object(&self, object: ObjectReference) { + debug!("Registering WB-unprotected object: {object}"); + let mut objects = self.wb_unprotected_objects.lock().unwrap(); + objects.insert(object); + } + + pub fn object_wb_unprotected_p(&self, object: ObjectReference) -> bool { + let objects = self.wb_unprotected_objects.lock().unwrap(); + objects.contains(&object) + } +} diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs new file mode 100644 index 0000000000..648efa4e27 --- /dev/null +++ b/gc/mmtk/src/collection.rs @@ -0,0 +1,122 @@ +use crate::abi::GCThreadTLS; + +use crate::api::RubyMutator; +use crate::heap::CpuHeapTrigger; +use crate::heap::RubyHeapTrigger; +use crate::heap::CPU_HEAP_TRIGGER_CONFIG; +use crate::mmtk; +use crate::upcalls; +use crate::Ruby; +use mmtk::memory_manager; +use mmtk::scheduler::*; +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; +use mmtk::util::VMWorkerThread; +use mmtk::vm::Collection; +use mmtk::vm::GCThreadContext; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::thread; + +static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(false); + +pub struct VMCollection {} + +impl Collection<Ruby> for VMCollection { + fn is_collection_enabled() -> bool { + crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed) + } + + fn stop_all_mutators<F>(tls: VMWorkerThread, mut mutator_visitor: F) + where + F: FnMut(&'static mut mmtk::Mutator<Ruby>), + { + (upcalls().stop_the_world)(); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed); + (upcalls().before_updating_jit_code)(); + } else { + CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed); + } + + crate::binding().pinning_registry.pin_children(tls); + + (upcalls().get_mutators)( + Self::notify_mutator_ready::<F>, + &mut mutator_visitor as *mut F as *mut _, + ); + } + + fn resume_mutators(_tls: VMWorkerThread) { + let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed); + + if current_gc_may_move { + (upcalls().after_updating_jit_code)(); + } + + (upcalls().resume_mutators)(current_gc_may_move); + } + + fn block_for_gc(tls: VMMutatorThread) { + (upcalls().block_for_gc)(tls); + } + + fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext<Ruby>) { + let join_handle = match ctx { + GCThreadContext::Worker(mut worker) => thread::Builder::new() + .name("MMTk Worker Thread".to_string()) + .spawn(move || { + let ordinal = worker.ordinal; + debug!("Hello! This is MMTk Worker Thread running! ordinal: {ordinal}"); + crate::register_gc_thread(thread::current().id()); + let ptr_worker = &mut *worker as *mut GCWorker<Ruby>; + let gc_thread_tls = + Box::into_raw(Box::new(GCThreadTLS::for_worker(ptr_worker))); + (upcalls().init_gc_worker_thread)(gc_thread_tls); + memory_manager::start_worker( + mmtk(), + GCThreadTLS::to_vwt(gc_thread_tls), + worker, + ); + debug!("An MMTk Worker Thread is quitting. Good bye! ordinal: {ordinal}"); + crate::unregister_gc_thread(thread::current().id()); + }) + .unwrap(), + }; + + { + let mut handles = crate::binding().gc_thread_join_handles.lock().unwrap(); + handles.push(join_handle); + } + } + + fn vm_live_bytes() -> usize { + (upcalls().vm_live_bytes)() + } + + fn create_gc_trigger() -> Box<dyn GCTriggerPolicy<Ruby>> { + // `GCTriggerSelector::Delegated` is currently used by two different + // heap modes: `ruby` (the Ruby-like free-slot ratio trigger) and `cpu` + // (the CPU-overhead trigger from Tavakolisomeh et al., MPLR '23). + // Which one is active is determined by which `OnceCell` config the + // `MMTK_HEAP_MODE` parser populated. + if CPU_HEAP_TRIGGER_CONFIG.get().is_some() { + Box::new(CpuHeapTrigger::default()) + } else { + Box::new(RubyHeapTrigger::default()) + } + } +} + +impl VMCollection { + extern "C" fn notify_mutator_ready<F>(mutator_ptr: *mut RubyMutator, data: *mut libc::c_void) + where + F: FnMut(&'static mut mmtk::Mutator<Ruby>), + { + let mutator = unsafe { &mut *mutator_ptr }; + let mutator_visitor = unsafe { &mut *(data as *mut F) }; + mutator_visitor(mutator); + } +} diff --git a/gc/mmtk/src/heap/cpu_heap_trigger.rs b/gc/mmtk/src/heap/cpu_heap_trigger.rs new file mode 100644 index 0000000000..ef5a79fe9a --- /dev/null +++ b/gc/mmtk/src/heap/cpu_heap_trigger.rs @@ -0,0 +1,370 @@ +//! A GC trigger that adjusts the heap size based on the CPU overhead of GC. +//! +//! This is an implementation of the heap sizing policy described in +//! Tavakolisomeh, Shimchenko, Österlund, Bruno, Ferreira, Wrigstad, +//! "Heap Size Adjustment with CPU Control", MPLR '23. +//! <https://doi.org/10.1145/3617651.3622988> +//! +//! The idea: rather than letting heap size control GC frequency, let a +//! user-supplied *target GC CPU overhead* control the heap size. After each GC +//! cycle, we measure the GC CPU overhead (fraction of process CPU time spent +//! in GC) and compare it to the target. If GC is over budget we grow the heap +//! (reducing GC frequency); if it is under budget we shrink the heap (trading +//! memory for more frequent collections). +//! +//! ## Algorithm +//! +//! After each GC cycle we compute, using an average of the last `n` cycles: +//! +//! ```text +//! GC_CPU = T_GC / T_APP (Eq. 1) +//! overhead_error = GC_CPU - target (Eq. 2) +//! sigmoid_error = 1 / (1 + e^(-overhead_error)) (Eq. 3) +//! adjustment_factor = sigmoid_error + 0.5 (in (0.5, 1.5)) (Eq. 4) +//! new_size = current_size * adjustment_factor (Eq. 5) +//! ``` +//! +//! where: +//! - `T_GC` is the wall-clock duration of each GC cycle. +//! - `T_APP` is process CPU time elapsed between consecutive GC cycles (sum of +//! CPU time over all threads — mutators, GC workers, compilers, etc.), read +//! via `clock_gettime(CLOCK_PROCESS_CPUTIME_ID)`. +//! +//! The final heap size is then clamped to the range +//! `[max(1.1 * used, min_heap_pages), max_heap_pages]`, providing 10% headroom +//! above current live memory to avoid triggering GC on an effectively-empty +//! heap. +//! +//! ## Differences from the paper +//! +//! The paper targets ZGC, a concurrent generational collector. MMTk's Ruby +//! binding currently ships stop-the-world collectors (Immix, MarkSweep). The +//! paper's formula still applies: with a STW collector the process CPU time +//! during GC closely tracks the wall-clock GC time, and mutator CPU time +//! during the mutator phase is correctly attributed. For generational plans +//! we skip nursery-only GCs, consistent with MemBalancer. + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::heap::SpaceStats; +use mmtk::Plan; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +use crate::Ruby; + +pub static CPU_HEAP_TRIGGER_CONFIG: OnceCell<CpuHeapTriggerConfig> = OnceCell::new(); + +/// Configuration for the [`CpuHeapTrigger`]. +pub struct CpuHeapTriggerConfig { + /// Lower bound on heap size (in pages). The trigger will never shrink below + /// this value. + pub min_heap_pages: usize, + /// Upper bound on heap size (in pages). The trigger will never grow above + /// this value. + pub max_heap_pages: usize, + /// Initial heap size (in pages). + pub initial_heap_pages: usize, + /// Target GC CPU overhead as a fraction of total process CPU time. For + /// example, `0.15` means the policy will try to keep GC CPU usage near 15%. + /// Valid range: `(0.0, 1.0)`. + pub target_gc_cpu: f64, + /// Number of recent GC cycles averaged together when computing the CPU + /// overhead signal. Smoothes out short-term fluctuations. The paper uses 3. + pub window_size: usize, +} + +/// A single GC cycle's timing measurements. +#[derive(Clone, Copy, Debug, Default)] +struct GcSample { + /// Wall-clock seconds spent inside this GC cycle. + gc_seconds: f64, + /// Seconds of process CPU time elapsed since the previous GC cycle ended. + /// This covers both mutator time and (on multi-threaded mutators) any + /// mutator CPU time consumed in parallel with the previous GC. + app_cpu_seconds: f64, +} + +struct CpuHeapTriggerState { + /// Ring buffer of the last `window_size` samples. Oldest-first. + samples: Vec<GcSample>, + /// Wall-clock time when the current GC cycle started. `None` when no GC is + /// in progress. + gc_start_wall: Option<std::time::Instant>, + /// Process CPU time (seconds) recorded at the end of the previous GC + /// cycle. `None` until the first cycle completes. + last_gc_end_cpu: Option<f64>, +} + +impl CpuHeapTriggerState { + fn new() -> Self { + Self { + samples: Vec::new(), + gc_start_wall: None, + last_gc_end_cpu: None, + } + } + + /// Pushes a new sample, dropping the oldest when the window is full. + fn push_sample(&mut self, sample: GcSample, window_size: usize) { + if self.samples.len() >= window_size { + self.samples.remove(0); + } + self.samples.push(sample); + } + + /// Returns the arithmetic mean GC CPU overhead across the window, or + /// `None` if we don't yet have a full sample (which happens on the first + /// GC cycle — we have no baseline for `app_cpu_seconds`). + fn mean_gc_cpu(&self) -> Option<f64> { + if self.samples.is_empty() { + return None; + } + let total_gc: f64 = self.samples.iter().map(|s| s.gc_seconds).sum(); + let total_app: f64 = self.samples.iter().map(|s| s.app_cpu_seconds).sum(); + if total_app <= 0.0 { + return None; + } + Some(total_gc / total_app) + } +} + +pub struct CpuHeapTrigger { + /// Target heap size in pages. Updated at the end of each GC cycle. + target_heap_pages: AtomicUsize, + /// Mutable timing state. Wrapped in a `Mutex` because `on_gc_start` and + /// `on_gc_end` are the only mutation sites and they are not on an + /// allocation hot path; avoiding the complexity of lock-free state is + /// worth the trivial contention. + state: Mutex<CpuHeapTriggerState>, +} + +impl Default for CpuHeapTrigger { + fn default() -> Self { + let cfg = Self::get_config(); + Self { + target_heap_pages: AtomicUsize::new(cfg.initial_heap_pages), + state: Mutex::new(CpuHeapTriggerState::new()), + } + } +} + +impl GCTriggerPolicy<Ruby> for CpuHeapTrigger { + fn is_gc_required( + &self, + space_full: bool, + space: Option<SpaceStats<Ruby>>, + plan: &dyn Plan<VM = Ruby>, + ) -> bool { + // Let the plan decide, matching the other triggers. + plan.collection_required(space_full, space) + } + + fn on_gc_start(&self, _mmtk: &'static MMTK<Ruby>) { + let mut state = self.state.lock().unwrap(); + state.gc_start_wall = Some(std::time::Instant::now()); + } + + fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) { + // Skip nursery-only GCs for generational plans. The heap resizing + // decision is driven by the (much more expensive) full collections + // where the signal-to-noise ratio is high enough to be useful. + if let Some(gen_plan) = mmtk.get_plan().generational() { + if gen_plan.is_current_gc_nursery() { + return; + } + } + + let cfg = Self::get_config(); + let gc_end_cpu = process_cpu_time_seconds(); + + let mut state = self.state.lock().unwrap(); + + // Duration of this GC cycle (wall clock). + let gc_seconds = state + .gc_start_wall + .take() + .map(|start| start.elapsed().as_secs_f64()) + .unwrap_or(0.0); + + // Process CPU time elapsed since the previous GC cycle ended. We + // require at least one previous end timestamp to produce a valid + // sample — without it we cannot compute `T_APP`. + if let (Some(last_end), Some(now)) = (state.last_gc_end_cpu, gc_end_cpu) { + let app_cpu_seconds = (now - last_end).max(0.0); + // Only record non-degenerate samples to avoid poisoning the window + // with zero-time entries from back-to-back GCs. + if app_cpu_seconds > 0.0 { + state.push_sample( + GcSample { + gc_seconds, + app_cpu_seconds, + }, + cfg.window_size, + ); + } + } + state.last_gc_end_cpu = gc_end_cpu; + + // Compute the new heap size only when we have samples to average over. + if let Some(gc_cpu) = state.mean_gc_cpu() { + // Drop the lock before doing the (relatively cheap) math and + // atomic update; nothing below needs the state. + drop(state); + + let overhead_error = gc_cpu - cfg.target_gc_cpu; // Eq. (2) + let sigmoid_error = sigmoid(overhead_error); // Eq. (3) + let adjustment_factor = sigmoid_error + 0.5; // Eq. (4), range (0.5, 1.5) + + let current = self.target_heap_pages.load(Ordering::Relaxed); + let suggested = ((current as f64) * adjustment_factor) as usize; // Eq. (5) + + // Clamp: + // - upper bound: configured max + // - lower bound: max(1.1 * used, min) — 10% headroom above current + // live memory, so we never request a heap so small that GC is + // triggered immediately on return from this one. + let used = mmtk.get_plan().get_used_pages(); + let floor = ((used as f64) * 1.1).ceil() as usize; + let lower = floor.max(cfg.min_heap_pages).min(cfg.max_heap_pages); + let upper = cfg.max_heap_pages; + let new_target = suggested.clamp(lower, upper); + + self.target_heap_pages.store(new_target, Ordering::Relaxed); + + info!( + "CpuHeapTrigger: gc_cpu={:.4} target={:.4} factor={:.4} \ + pages {} -> {} (used={}, clamp=[{}, {}])", + gc_cpu, + cfg.target_gc_cpu, + adjustment_factor, + current, + new_target, + used, + lower, + upper + ); + } + } + + fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool { + plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_current_heap_size_in_pages(&self) -> usize { + self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_max_heap_size_in_pages(&self) -> usize { + Self::get_config().max_heap_pages + } + + fn can_heap_size_grow(&self) -> bool { + self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages + } +} + +impl CpuHeapTrigger { + fn get_config<'b>() -> &'b CpuHeapTriggerConfig { + CPU_HEAP_TRIGGER_CONFIG + .get() + .expect("Attempt to use CPU_HEAP_TRIGGER_CONFIG before it is initialized") + } +} + +/// Standard logistic sigmoid. Returns 0.5 when x == 0, asymptotes to 0 and 1. +fn sigmoid(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) +} + +/// Reads the process-wide CPU time as a floating-point number of seconds, +/// summed across all threads of this process. Returns `None` if the clock +/// query fails (which should be essentially impossible on supported +/// platforms). +fn process_cpu_time_seconds() -> Option<f64> { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: `clock_gettime` writes exactly `sizeof(timespec)` bytes to the + // pointer we pass, which is a valid local stack allocation. + let rc = unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) }; + if rc != 0 { + return None; + } + Some((ts.tv_sec as f64) + (ts.tv_nsec as f64) / 1_000_000_000.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sigmoid_is_well_behaved() { + assert!((sigmoid(0.0) - 0.5).abs() < 1e-12); + assert!(sigmoid(-100.0) < 1e-9); + assert!(sigmoid(100.0) > 1.0 - 1e-9); + // Monotonic. + assert!(sigmoid(-1.0) < sigmoid(0.0)); + assert!(sigmoid(0.0) < sigmoid(1.0)); + } + + #[test] + fn adjustment_factor_is_within_paper_bounds() { + // Eq. (4): adjustment_factor = sigmoid(e) + 0.5 must lie in (0.5, 1.5). + for e in [-10.0_f64, -1.0, 0.0, 1.0, 10.0] { + let f = sigmoid(e) + 0.5; + assert!(f > 0.5 && f < 1.5, "factor {f} out of range for e={e}"); + } + } + + #[test] + fn mean_gc_cpu_is_total_weighted() { + let mut state = CpuHeapTriggerState::new(); + state.push_sample( + GcSample { + gc_seconds: 1.0, + app_cpu_seconds: 10.0, + }, + 3, + ); + state.push_sample( + GcSample { + gc_seconds: 3.0, + app_cpu_seconds: 10.0, + }, + 3, + ); + // (1 + 3) / (10 + 10) = 0.2 + assert!((state.mean_gc_cpu().unwrap() - 0.2).abs() < 1e-12); + } + + #[test] + fn window_drops_oldest() { + let mut state = CpuHeapTriggerState::new(); + for i in 0..5 { + state.push_sample( + GcSample { + gc_seconds: i as f64, + app_cpu_seconds: 1.0, + }, + 3, + ); + } + assert_eq!(state.samples.len(), 3); + // After pushing 0,1,2,3,4 with window 3, we should have [2,3,4]. + assert_eq!(state.samples[0].gc_seconds, 2.0); + assert_eq!(state.samples[2].gc_seconds, 4.0); + } + + #[test] + fn no_sample_without_prior_gc() { + // First GC cycle cannot produce a sample (no previous end time). The + // push happens only when last_gc_end_cpu is Some. + let state = CpuHeapTriggerState::new(); + assert!(state.mean_gc_cpu().is_none()); + } +} diff --git a/gc/mmtk/src/heap/mod.rs b/gc/mmtk/src/heap/mod.rs new file mode 100644 index 0000000000..05a35efb23 --- /dev/null +++ b/gc/mmtk/src/heap/mod.rs @@ -0,0 +1,9 @@ +mod cpu_heap_trigger; +mod ruby_heap_trigger; + +pub use cpu_heap_trigger::CpuHeapTrigger; +pub use cpu_heap_trigger::CpuHeapTriggerConfig; +pub use cpu_heap_trigger::CPU_HEAP_TRIGGER_CONFIG; +pub use ruby_heap_trigger::RubyHeapTrigger; +pub use ruby_heap_trigger::RubyHeapTriggerConfig; +pub use ruby_heap_trigger::RUBY_HEAP_TRIGGER_CONFIG; diff --git a/gc/mmtk/src/heap/ruby_heap_trigger.rs b/gc/mmtk/src/heap/ruby_heap_trigger.rs new file mode 100644 index 0000000000..fe1130043d --- /dev/null +++ b/gc/mmtk/src/heap/ruby_heap_trigger.rs @@ -0,0 +1,105 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::heap::SpaceStats; +use mmtk::Plan; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +use crate::Ruby; + +pub static RUBY_HEAP_TRIGGER_CONFIG: OnceCell<RubyHeapTriggerConfig> = OnceCell::new(); + +pub struct RubyHeapTriggerConfig { + /// Min heap size + pub min_heap_pages: usize, + /// Max heap size + pub max_heap_pages: usize, + /// Minimum ratio of empty space after a GC before the heap will grow + pub heap_pages_min_ratio: f64, + /// Ratio the heap will grow by + pub heap_pages_goal_ratio: f64, + /// Maximum ratio of empty space after a GC before the heap will shrink + pub heap_pages_max_ratio: f64, +} + +pub struct RubyHeapTrigger { + /// Target number of heap pages + target_heap_pages: AtomicUsize, +} + +impl GCTriggerPolicy<Ruby> for RubyHeapTrigger { + fn is_gc_required( + &self, + space_full: bool, + space: Option<SpaceStats<Ruby>>, + plan: &dyn Plan<VM = Ruby>, + ) -> bool { + // Let the plan decide + plan.collection_required(space_full, space) + } + + fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) { + if let Some(plan) = mmtk.get_plan().generational() { + if plan.is_current_gc_nursery() { + // Nursery GC + } else { + // Full GC + } + + panic!("TODO: support for generational GC not implemented") + } else { + let used_pages = mmtk.get_plan().get_used_pages(); + + let target_min = + (used_pages as f64 * (1.0 + Self::get_config().heap_pages_min_ratio)) as usize; + let target_max = + (used_pages as f64 * (1.0 + Self::get_config().heap_pages_max_ratio)) as usize; + let new_target = + (((used_pages as f64) * (1.0 + Self::get_config().heap_pages_goal_ratio)) as usize) + .clamp( + Self::get_config().min_heap_pages, + Self::get_config().max_heap_pages, + ); + + if used_pages < target_min || used_pages > target_max { + self.target_heap_pages.store(new_target, Ordering::Relaxed); + } + } + } + + fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool { + plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_current_heap_size_in_pages(&self) -> usize { + self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_max_heap_size_in_pages(&self) -> usize { + Self::get_config().max_heap_pages + } + + fn can_heap_size_grow(&self) -> bool { + self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages + } +} + +impl Default for RubyHeapTrigger { + fn default() -> Self { + let min_heap_pages = Self::get_config().min_heap_pages; + + Self { + target_heap_pages: AtomicUsize::new(min_heap_pages), + } + } +} + +impl RubyHeapTrigger { + fn get_config<'b>() -> &'b RubyHeapTriggerConfig { + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("Attempt to use RUBY_HEAP_TRIGGER_CONFIG before it is initialized") + } +} diff --git a/gc/mmtk/src/lib.rs b/gc/mmtk/src/lib.rs new file mode 100644 index 0000000000..52dc782051 --- /dev/null +++ b/gc/mmtk/src/lib.rs @@ -0,0 +1,161 @@ +// Warn about unsafe operations in functions that are already marked as unsafe. +// This will become default in Rust 2024 edition. +#![warn(unsafe_op_in_unsafe_fn)] + +extern crate libc; +extern crate mmtk; +#[macro_use] +extern crate log; +extern crate probe; + +use std::collections::HashSet; +use std::panic::PanicHookInfo; +use std::sync::Mutex; +use std::thread::ThreadId; + +use abi::RubyUpcalls; +use binding::RubyBinding; +use binding::RubyBindingFast; +use binding::RubyConfiguration; +use mmtk::vm::slot::SimpleSlot; +use mmtk::vm::slot::UnimplementedMemorySlice; +use mmtk::vm::VMBinding; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +pub mod abi; +pub mod active_plan; +pub mod api; +pub mod binding; +pub mod collection; +pub mod heap; +pub mod object_model; +pub mod pinning_registry; +pub mod reference_glue; +pub mod scanning; +pub mod utils; +pub mod weak_proc; + +#[derive(Default)] +pub struct Ruby; + +/// Ruby slot type, i.e. a slot that holds a VALUE. +/// Currently we use SimpleSlot. +/// It doesn't matter, becaues we have not started using slot-enqueuing, yet. +pub type RubySlot = SimpleSlot; + +/// Ruby memory slice, i.e. an array of VALUEs. +/// It is used by array-copy barriers which is supposed to perform bettern than copying array +/// elements one by one. At this moment, we just leave it unimplemented. +pub type RubyMemorySlice = UnimplementedMemorySlice<RubySlot>; + +impl VMBinding for Ruby { + type VMObjectModel = object_model::VMObjectModel; + type VMScanning = scanning::VMScanning; + type VMCollection = collection::VMCollection; + type VMActivePlan = active_plan::VMActivePlan; + type VMReferenceGlue = reference_glue::VMReferenceGlue; + + type VMSlot = RubySlot; + type VMMemorySlice = RubyMemorySlice; +} + +/// The callback for mutator thread panic handler (which calls rb_bug to output +/// debugging information such as the Ruby backtrace and memory maps). +/// This is set before BINDING is set because mmtk_init could panic. +pub static MUTATOR_THREAD_PANIC_HANDLER: OnceCell<extern "C" fn()> = OnceCell::new(); + +/// The singleton object for the Ruby binding itself. +pub static BINDING: OnceCell<RubyBinding> = OnceCell::new(); + +/// Some data needs to be accessed fast. +/// We sacrifice safety for speed using unsynchronized global variables. +pub static mut BINDING_FAST: RubyBindingFast = RubyBindingFast::new(); + +/// Some data needs to be accessed fast. +pub static CONFIGURATION: RubyConfiguration = RubyConfiguration::new(); + +pub fn binding<'b>() -> &'b RubyBinding { + BINDING + .get() + .expect("Attempt to use the binding before it is initialization") +} + +pub fn mmtk() -> &'static MMTK<Ruby> { + binding().mmtk +} + +pub fn upcalls() -> &'static RubyUpcalls { + binding().upcalls() +} + +pub static GC_THREADS: OnceCell<Mutex<HashSet<ThreadId>>> = OnceCell::new(); + +pub(crate) fn register_gc_thread(thread_id: ThreadId) { + let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.insert(thread_id); +} + +pub(crate) fn unregister_gc_thread(thread_id: ThreadId) { + let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.remove(&thread_id); +} + +pub(crate) fn is_gc_thread(thread_id: ThreadId) -> bool { + let gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.contains(&thread_id) +} + +fn handle_gc_thread_panic(panic_info: &PanicHookInfo) { + eprintln!("ERROR: An MMTk GC thread panicked. This is a bug."); + eprintln!("{panic_info}"); + + let bt = std::backtrace::Backtrace::capture(); + match bt.status() { + std::backtrace::BacktraceStatus::Unsupported => { + eprintln!("Backtrace is unsupported.") + } + std::backtrace::BacktraceStatus::Disabled => { + eprintln!("Backtrace is disabled."); + eprintln!("run with `RUST_BACKTRACE=1` environment variable to display a backtrace"); + } + std::backtrace::BacktraceStatus::Captured => { + eprintln!("{bt}"); + } + s => { + eprintln!("Unknown backtrace status: {s:?}"); + } + } +} + +pub(crate) fn set_panic_hook() { + if GC_THREADS.set(Default::default()).is_err() { + return; + } + + let old_hook = std::panic::take_hook(); + + std::panic::set_hook(Box::new(move |panic_info| { + if is_gc_thread(std::thread::current().id()) { + handle_gc_thread_panic(panic_info); + + (crate::binding().upcalls().gc_thread_panic_handler)(); + } else { + old_hook(panic_info); + (crate::MUTATOR_THREAD_PANIC_HANDLER + .get() + .expect("MUTATOR_THREAD_PANIC_HANDLER is not set"))(); + } + })); +} + +/// This kind of assertion is enabled if either building in debug mode or the +/// "extra_assert" feature is enabled. +#[macro_export] +macro_rules! extra_assert { + ($($arg:tt)*) => { + if std::cfg!(any(debug_assertions, feature = "extra_assert")) { + std::assert!($($arg)*); + } + }; +} diff --git a/gc/mmtk/src/object_model.rs b/gc/mmtk/src/object_model.rs new file mode 100644 index 0000000000..d673ca11a0 --- /dev/null +++ b/gc/mmtk/src/object_model.rs @@ -0,0 +1,124 @@ +use std::ptr::copy_nonoverlapping; + +use crate::abi; +use crate::abi::RubyObjectAccess; +use crate::abi::MIN_OBJ_ALIGN; +use crate::abi::OBJREF_OFFSET; +use crate::Ruby; +use mmtk::util::constants::BITS_IN_BYTE; +use mmtk::util::copy::CopySemantics; +use mmtk::util::copy::GCWorkerCopyContext; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::vm::*; + +pub struct VMObjectModel {} + +impl VMObjectModel { + const OBJREF_OFFSET: usize = abi::OBJREF_OFFSET; +} + +impl ObjectModel<Ruby> for VMObjectModel { + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); + + // We overwrite the prepended word which were used to hold object sizes. + const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = + VMLocalForwardingPointerSpec::in_header(-((OBJREF_OFFSET * BITS_IN_BYTE) as isize)); + + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = + VMLocalForwardingBitsSpec::side_first(); + + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = + VMLocalMarkBitSpec::side_after(Self::LOCAL_FORWARDING_BITS_SPEC.as_spec()); + + const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = + VMLocalPinningBitSpec::side_after(Self::LOCAL_MARK_BIT_SPEC.as_spec()); + + const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = + VMLocalLOSMarkNurserySpec::side_after(Self::LOCAL_PINNING_BIT_SPEC.as_spec()); + + const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false; + const OBJECT_REF_OFFSET_LOWER_BOUND: isize = Self::OBJREF_OFFSET as isize; + + const NEED_VO_BITS_DURING_TRACING: bool = true; + + fn copy( + from: ObjectReference, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext<Ruby>, + ) -> ObjectReference { + let from_acc = RubyObjectAccess::from_objref(from); + let from_start = from_acc.obj_start(); + let object_size = from_acc.object_size(); + let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics); + debug_assert!(!to_start.is_zero()); + let to_payload = to_start.add(OBJREF_OFFSET); + unsafe { + copy_nonoverlapping::<u8>(from_start.to_ptr(), to_start.to_mut_ptr(), object_size); + } + let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(to_payload) }; + copy_context.post_copy(to_obj, object_size, semantics); + trace!("Copied object from {} to {}", from, to_obj); + + (crate::binding().upcalls().move_obj_during_marking)(from, to_obj); + + #[cfg(feature = "clear_old_copy")] + { + trace!( + "Clearing old copy {} ({}-{})", + from, + from_start, + from_start + object_size + ); + // For debug purpose, we clear the old copy so that if the Ruby VM reads from the old + // copy again, it will likely result in an error. + unsafe { std::ptr::write_bytes::<u8>(from_start.to_mut_ptr(), 0, object_size) } + } + + to_obj + } + + fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address { + unimplemented!( + "This function cannot be called because we do not support MarkCompact for Ruby." + ) + } + + fn get_reference_when_copied_to(_from: ObjectReference, _to: Address) -> ObjectReference { + unimplemented!( + "This function cannot be called because we do not support MarkCompact for Ruby." + ) + } + + fn get_current_size(object: ObjectReference) -> usize { + RubyObjectAccess::from_objref(object).object_size() + } + + fn get_type_descriptor(_reference: ObjectReference) -> &'static [i8] { + todo!() + } + + fn ref_to_object_start(object: ObjectReference) -> Address { + RubyObjectAccess::from_objref(object).obj_start() + } + + fn ref_to_header(object: ObjectReference) -> Address { + RubyObjectAccess::from_objref(object).payload_addr() + } + + fn get_size_when_copied(object: ObjectReference) -> usize { + Self::get_current_size(object) + } + + fn get_align_when_copied(_object: ObjectReference) -> usize { + todo!() + } + + fn get_align_offset_when_copied(_object: ObjectReference) -> usize { + todo!() + } + + fn dump_object(_object: ObjectReference) { + todo!() + } +} diff --git a/gc/mmtk/src/pinning_registry.rs b/gc/mmtk/src/pinning_registry.rs new file mode 100644 index 0000000000..b498b508f1 --- /dev/null +++ b/gc/mmtk/src/pinning_registry.rs @@ -0,0 +1,187 @@ +use std::sync::Mutex; + +use mmtk::memory_manager; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::MMTK; + +use crate::abi::GCThreadTLS; +use crate::upcalls; +use crate::Ruby; + +pub struct PinningRegistry { + pinning_objs: Mutex<Vec<ObjectReference>>, + pinned_objs: Mutex<Vec<ObjectReference>>, +} + +impl PinningRegistry { + pub fn new() -> Self { + Self { + pinning_objs: Default::default(), + pinned_objs: Default::default(), + } + } + + pub fn register(&self, object: ObjectReference) { + let mut pinning_objs = self.pinning_objs.lock().unwrap(); + pinning_objs.push(object); + } + + pub fn pin_children(&self, tls: VMWorkerThread) { + if !crate::mmtk().get_plan().current_gc_may_move_object() { + log::debug!("The current GC is non-moving, skipping pinning children."); + return; + } + + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let worker = gc_tls.worker(); + + let pinning_objs = self + .pinning_objs + .try_lock() + .expect("PinningRegistry should not have races during GC."); + + let packet_size = 512; + let work_packets = pinning_objs + .chunks(packet_size) + .map(|chunk| { + Box::new(PinPinningChildren { + pinning_objs: chunk.to_vec(), + }) as _ + }) + .collect(); + + worker.scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(work_packets); + } + + pub fn cleanup(&self, worker: &mut GCWorker<Ruby>) { + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(RemoveDeadPinnings); + if crate::mmtk().get_plan().current_gc_may_move_object() { + let packet = { + let mut pinned_objs = self + .pinned_objs + .try_lock() + .expect("Unexpected contention on pinned_objs"); + UnpinPinnedObjects { + objs: std::mem::take(&mut pinned_objs), + } + }; + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(packet); + } else { + debug!("The current GC is non-moving, skipping unpinning objects."); + debug_assert_eq!( + { + let pinned_objs = self + .pinned_objs + .try_lock() + .expect("Unexpected contention on pinned_objs"); + pinned_objs.len() + }, + 0 + ); + } + } +} + +impl Default for PinningRegistry { + fn default() -> Self { + Self::new() + } +} + +struct PinPinningChildren { + pinning_objs: Vec<ObjectReference>, +} + +impl GCWork<Ruby> for PinPinningChildren { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + let mut pinned_objs = vec![]; + let mut newly_pinned_objs = vec![]; + + let visit_object = |_worker, target_object: ObjectReference, pin| { + log::trace!( + " -> {} {}", + if pin { "(pin)" } else { " " }, + target_object + ); + if pin { + debug_assert!( + target_object.get_forwarded_object().is_none(), + "Trying to pin {target_object} but has been moved" + ); + + pinned_objs.push(target_object); + } + target_object + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + for obj in self.pinning_objs.iter().cloned() { + log::trace!(" Pinning: {}", obj); + (upcalls().call_gc_mark_children)(obj); + } + }); + + for target_object in pinned_objs { + if memory_manager::pin_object(target_object) { + newly_pinned_objs.push(target_object); + } + } + + let mut pinned_objs = crate::binding() + .pinning_registry + .pinned_objs + .lock() + .unwrap(); + pinned_objs.append(&mut newly_pinned_objs); + } +} + +struct RemoveDeadPinnings; + +impl GCWork<Ruby> for RemoveDeadPinnings { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + log::debug!("Removing dead Pinnings..."); + + let registry = &crate::binding().pinning_registry; + { + let mut pinning_objs = registry + .pinning_objs + .try_lock() + .expect("PinningRegistry should not have races during GC."); + + pinning_objs.retain_mut(|obj| { + if obj.is_live() { + let new_obj = obj.get_forwarded_object().unwrap_or(*obj); + *obj = new_obj; + true + } else { + log::trace!(" Dead Pinning removed: {}", *obj); + false + } + }); + } + } +} + +struct UnpinPinnedObjects { + objs: Vec<ObjectReference>, +} + +impl GCWork<Ruby> for UnpinPinnedObjects { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + log::debug!("Unpinning pinned objects..."); + + for obj in self.objs.iter() { + let unpinned = memory_manager::unpin_object(*obj); + debug_assert!(unpinned); + } + } +} diff --git a/gc/mmtk/src/reference_glue.rs b/gc/mmtk/src/reference_glue.rs new file mode 100644 index 0000000000..1272bd54c1 --- /dev/null +++ b/gc/mmtk/src/reference_glue.rs @@ -0,0 +1,26 @@ +use crate::Ruby; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::vm::ReferenceGlue; + +pub struct VMReferenceGlue {} + +impl ReferenceGlue<Ruby> for VMReferenceGlue { + type FinalizableType = ObjectReference; + + fn get_referent(_object: ObjectReference) -> Option<ObjectReference> { + unimplemented!() + } + + fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { + unimplemented!() + } + + fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) { + unimplemented!() + } + + fn clear_referent(_new_reference: ObjectReference) { + unimplemented!() + } +} diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs new file mode 100644 index 0000000000..355a2e7759 --- /dev/null +++ b/gc/mmtk/src/scanning.rs @@ -0,0 +1,291 @@ +use crate::abi::GCThreadTLS; + +use crate::upcalls; +use crate::utils::ChunkedVecCollector; +use crate::Ruby; +use crate::RubySlot; +use mmtk::memory_manager; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::vm::ObjectTracer; +use mmtk::vm::RootsWorkFactory; +use mmtk::vm::Scanning; +use mmtk::vm::SlotVisitor; +use mmtk::Mutator; + +pub struct VMScanning {} + +impl Scanning<Ruby> for VMScanning { + const UNIQUE_OBJECT_ENQUEUING: bool = true; + + fn support_slot_enqueuing(_tls: VMWorkerThread, _object: ObjectReference) -> bool { + false + } + + fn scan_object<EV: SlotVisitor<RubySlot>>( + _tls: VMWorkerThread, + _object: ObjectReference, + _slot_visitor: &mut EV, + ) { + unreachable!("We have not enabled slot enqueuing for any types, yet."); + } + + fn scan_object_and_trace_edges<OT: ObjectTracer>( + tls: VMWorkerThread, + object: ObjectReference, + object_tracer: &mut OT, + ) { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "Not an MMTk object: {object}", + ); + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let visit_object = |_worker, target_object: ObjectReference, pin| { + trace!( + "Tracing edge: {} -> {}{}", + object, + target_object, + if pin { " pin" } else { "" } + ); + debug_assert!( + mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(), + "Destination is not an MMTk object. Src: {object} dst: {target_object}" + ); + + debug_assert!( + // If we are in a moving GC, all objects should be pinned by PinningRegistry. + // If it is requested that target_object be pinned but it is not pinned, then + // it is a bug because it could be moved. + if crate::mmtk().get_plan().current_gc_may_move_object() && pin { + memory_manager::is_pinned(target_object) + } else { + true + }, + "Object {object} is trying to pin {target_object}" + ); + + let forwarded_target = object_tracer.trace_object(target_object); + if forwarded_target != target_object { + trace!(" Forwarded target {target_object} -> {forwarded_target}"); + } + forwarded_target + }; + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + (upcalls().call_gc_mark_children)(object); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + (upcalls().update_object_references)(object); + } + }); + } + + fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { + // Do nothing + } + + fn scan_roots_in_mutator_thread( + _tls: VMWorkerThread, + _mutator: &'static mut Mutator<Ruby>, + mut _factory: impl RootsWorkFactory<RubySlot>, + ) { + // Do nothing. All stacks (including Ruby stacks and machine stacks) are reachable from + // `rb_vm_t` -> ractor -> thread -> fiber -> stacks. It is part of `ScanGCRoots` which + // calls `rb_gc_mark_roots` -> `rb_vm_mark`. + } + + fn scan_vm_specific_roots(tls: VMWorkerThread, factory: impl RootsWorkFactory<RubySlot>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let root_scanning_work_packets: Vec<Box<dyn GCWork<Ruby>>> = vec![ + Box::new(ScanGCRoots::new(factory.clone())), + Box::new(ScanObjspace::new(factory.clone())), + ]; + gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare] + .bulk_add(root_scanning_work_packets); + + // Generate WB-unprotected roots scanning work packets + + 'gen_wb_unprotected_work: { + let is_nursery_gc = (crate::mmtk().get_plan().generational()) + .is_some_and(|gen| gen.is_current_gc_nursery()); + if !is_nursery_gc { + break 'gen_wb_unprotected_work; + } + + let vecs = { + let guard = crate::binding() + .wb_unprotected_objects + .try_lock() + .expect("Someone is holding the lock of wb_unprotected_objects?"); + if guard.is_empty() { + break 'gen_wb_unprotected_work; + } + + let mut collector = ChunkedVecCollector::new(128); + collector.extend(guard.iter().copied()); + collector.into_vecs() + }; + + let packets = vecs + .into_iter() + .map(|objects| { + let factory = factory.clone(); + Box::new(ScanWbUnprotectedRoots { factory, objects }) as _ + }) + .collect::<Vec<_>>(); + + gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(packets); + } + } + + fn supports_return_barrier() -> bool { + false + } + + fn prepare_for_roots_re_scanning() { + todo!() + } + + fn process_weak_refs( + worker: &mut GCWorker<Ruby>, + tracer_context: impl mmtk::vm::ObjectTracerContext<Ruby>, + ) -> bool { + crate::binding() + .weak_proc + .process_weak_stuff(worker, tracer_context); + crate::binding().pinning_registry.cleanup(worker); + false + } + + fn forward_weak_refs( + _worker: &mut GCWorker<Ruby>, + _tracer_context: impl mmtk::vm::ObjectTracerContext<Ruby>, + ) { + panic!("We can't use MarkCompact in Ruby."); + } +} + +impl VMScanning { + const OBJECT_BUFFER_SIZE: usize = 4096; + + fn collect_object_roots_in<F: FnOnce()>( + root_scan_kind: &str, + gc_tls: &mut GCThreadTLS, + factory: &mut impl RootsWorkFactory<RubySlot>, + callback: F, + ) { + let mut buffer: Vec<ObjectReference> = Vec::new(); + let visit_object = |_, object: ObjectReference, pin| { + debug!( + "[{}] Visiting object: {}{}", + root_scan_kind, + object, + if pin { + "(unmovable root)" + } else { + "(movable, but we pin it anyway)" + } + ); + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "Root does not point to MMTk object. object: {object}" + ); + buffer.push(object); + if buffer.len() >= Self::OBJECT_BUFFER_SIZE { + factory.create_process_pinning_roots_work(std::mem::take(&mut buffer)); + } + object + }; + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, callback); + + if !buffer.is_empty() { + factory.create_process_pinning_roots_work(buffer); + } + } +} + +trait GlobaRootScanningWork { + type F: RootsWorkFactory<RubySlot>; + const NAME: &'static str; + + fn new(factory: Self::F) -> Self; + fn scan_roots(); + fn roots_work_factory(&mut self) -> &mut Self::F; + + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + let factory = self.roots_work_factory(); + + VMScanning::collect_object_roots_in(Self::NAME, gc_tls, factory, || { + Self::scan_roots(); + }); + } +} + +macro_rules! define_global_root_scanner { + ($name: ident, $code: expr) => { + struct $name<F: RootsWorkFactory<RubySlot>> { + factory: F, + } + impl<F: RootsWorkFactory<RubySlot>> GlobaRootScanningWork for $name<F> { + type F = F; + const NAME: &'static str = stringify!($name); + fn new(factory: Self::F) -> Self { + Self { factory } + } + fn scan_roots() { + $code + } + fn roots_work_factory(&mut self) -> &mut Self::F { + &mut self.factory + } + } + impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for $name<F> { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobaRootScanningWork::do_work(self, worker, mmtk); + } + } + }; +} + +define_global_root_scanner!(ScanGCRoots, { + (crate::upcalls().scan_gc_roots)(); +}); + +define_global_root_scanner!(ScanObjspace, { + (crate::upcalls().scan_objspace)(); +}); + +struct ScanWbUnprotectedRoots<F: RootsWorkFactory<RubySlot>> { + factory: F, + objects: Vec<ObjectReference>, +} + +impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for ScanWbUnprotectedRoots<F> { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + VMScanning::collect_object_roots_in("wb_unprot_roots", gc_tls, &mut self.factory, || { + for object in self.objects.iter().copied() { + if object.is_reachable() { + debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}"); + (upcalls().call_gc_mark_children)(object); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + (upcalls().update_object_references)(object); + } + } else { + debug!( + "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {object}" + ); + } + } + }); + } +} diff --git a/gc/mmtk/src/utils.rs b/gc/mmtk/src/utils.rs new file mode 100644 index 0000000000..d1979eaf58 --- /dev/null +++ b/gc/mmtk/src/utils.rs @@ -0,0 +1,161 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use atomic_refcell::AtomicRefCell; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; + +use crate::Ruby; +use sysinfo::System; + +pub struct ChunkedVecCollector<T> { + vecs: Vec<Vec<T>>, + current_vec: Vec<T>, + chunk_size: usize, +} + +impl<T> ChunkedVecCollector<T> { + pub fn new(chunk_size: usize) -> Self { + Self { + vecs: vec![], + current_vec: Vec::with_capacity(chunk_size), + chunk_size, + } + } + + pub fn add(&mut self, item: T) { + self.current_vec.push(item); + if self.current_vec.len() == self.chunk_size { + self.flush(); + } + } + + fn flush(&mut self) { + let new_vec = Vec::with_capacity(self.chunk_size); + let old_vec = std::mem::replace(&mut self.current_vec, new_vec); + self.vecs.push(old_vec); + } + + pub fn into_vecs(mut self) -> Vec<Vec<T>> { + if !self.current_vec.is_empty() { + self.flush(); + } + self.vecs + } +} + +impl<A> Extend<A> for ChunkedVecCollector<A> { + fn extend<T: IntoIterator<Item = A>>(&mut self, iter: T) { + for item in iter { + self.add(item); + } + } +} + +pub struct AfterAll { + counter: AtomicUsize, + stage: WorkBucketStage, + packets: AtomicRefCell<Vec<Box<dyn GCWork<Ruby>>>>, +} + +unsafe impl Sync for AfterAll {} + +impl AfterAll { + pub fn new(stage: WorkBucketStage) -> Self { + Self { + counter: AtomicUsize::new(0), + stage, + packets: AtomicRefCell::new(vec![]), + } + } + + pub fn add_packets(&self, mut packets: Vec<Box<dyn GCWork<Ruby>>>) { + let mut borrow = self.packets.borrow_mut(); + borrow.append(&mut packets); + } + + pub fn count_up(&self, n: usize) { + self.counter.fetch_add(n, Ordering::SeqCst); + } + + pub fn count_down(&self, worker: &mut GCWorker<Ruby>) { + let old = self.counter.fetch_sub(1, Ordering::SeqCst); + if old == 1 { + let packets = { + let mut borrow = self.packets.borrow_mut(); + std::mem::take(borrow.as_mut()) + }; + worker.scheduler().work_buckets[self.stage].bulk_add(packets); + } + } +} + +pub fn default_heap_max() -> usize { + let mut s = System::new(); + s.refresh_memory(); + s.total_memory() + .checked_mul(80) + .and_then(|v| v.checked_div(100)) + .expect("Invalid Memory size") as usize +} + +pub fn parse_capacity(input: &str) -> Option<usize> { + let trimmed = input.trim(); + + const KIBIBYTE: usize = 1024; + const MEBIBYTE: usize = 1024 * KIBIBYTE; + const GIBIBYTE: usize = 1024 * MEBIBYTE; + + let (number, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) { + trimmed.split_at(pos) + } else { + (trimmed, "") + }; + + let Ok(v) = number.parse::<usize>() else { + return None; + }; + + match suffix { + "GiB" => Some(v * GIBIBYTE), + "MiB" => Some(v * MEBIBYTE), + "KiB" => Some(v * KIBIBYTE), + "" => Some(v), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_capacity_parses_bare_bytes() { + assert_eq!(Some(1234), parse_capacity("1234")); + } + + #[test] + fn test_parse_capacity_parses_kibibytes() { + assert_eq!(Some(10240), parse_capacity("10KiB")); + } + + #[test] + fn test_parse_capacity_parses_mebibytes() { + assert_eq!(Some(10485760), parse_capacity("10MiB")) + } + + #[test] + fn test_parse_capacity_parses_gibibytes() { + assert_eq!(Some(10737418240), parse_capacity("10GiB")) + } + + #[test] + fn test_parse_capacity_parses_nonsense_values() { + assert_eq!(None, parse_capacity("notanumber")); + assert_eq!(None, parse_capacity("5tartswithanumber")); + assert_eq!(None, parse_capacity("number1nthemiddle")); + assert_eq!(None, parse_capacity("numberattheend111")); + assert_eq!(None, parse_capacity("mult1pl3numb3r5")); + } +} diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs new file mode 100644 index 0000000000..d38dbe04a4 --- /dev/null +++ b/gc/mmtk/src/weak_proc.rs @@ -0,0 +1,328 @@ +use std::sync::Mutex; + +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::vm::ObjectTracerContext; + +use crate::abi::GCThreadTLS; +use crate::upcalls; +use crate::Ruby; + +pub struct WeakProcessor { + non_parallel_obj_free_candidates: Mutex<Vec<ObjectReference>>, + parallel_obj_free_candidates: Vec<Mutex<Vec<ObjectReference>>>, + + /// Objects that needs `obj_free` called when dying. + /// If it is a bottleneck, replace it with a lock-free data structure, + /// or add candidates in batch. + weak_references: Mutex<Vec<ObjectReference>>, +} + +impl Default for WeakProcessor { + fn default() -> Self { + Self::new() + } +} + +impl WeakProcessor { + pub fn new() -> Self { + Self { + non_parallel_obj_free_candidates: Mutex::new(Vec::new()), + parallel_obj_free_candidates: vec![Mutex::new(Vec::new())], + weak_references: Mutex::new(Vec::new()), + } + } + + pub fn init_parallel_obj_free_candidates(&mut self, num_workers: usize) { + debug_assert_eq!(self.parallel_obj_free_candidates.len(), 1); + + for _ in 1..num_workers { + self.parallel_obj_free_candidates + .push(Mutex::new(Vec::new())); + } + } + + /// Add a batch of objects as candidates for `obj_free`. + /// + /// Amortizes mutex acquisition over the entire batch. Called when a + /// mutator's local buffer is flushed (buffer full or stop-the-world). + pub fn add_obj_free_candidates_batch( + &self, + objects: &[ObjectReference], + can_parallel_free: bool, + ) { + if objects.is_empty() { + return; + } + + if can_parallel_free { + let num_buckets = self.parallel_obj_free_candidates.len(); + for idx in 0..num_buckets { + let mut bucket = self.parallel_obj_free_candidates[idx].lock().unwrap(); + for (i, &obj) in objects.iter().enumerate() { + if i % num_buckets == idx { + bucket.push(obj); + } + } + } + } else { + self.non_parallel_obj_free_candidates + .lock() + .unwrap() + .extend_from_slice(objects); + } + } + + pub fn get_all_obj_free_candidates(&self) -> Vec<ObjectReference> { + // let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap(); + let mut all_obj_free_candidates = self + .non_parallel_obj_free_candidates + .lock() + .unwrap() + .to_vec(); + + for candidates_mutex in &self.parallel_obj_free_candidates { + all_obj_free_candidates.extend(candidates_mutex.lock().unwrap().to_vec()); + } + + std::mem::take(all_obj_free_candidates.as_mut()) + } + + pub fn add_weak_reference(&self, object: ObjectReference) { + let mut weak_references = self.weak_references.lock().unwrap(); + weak_references.push(object); + } + + pub fn weak_references_count(&self) -> usize { + self.weak_references.lock().unwrap().len() + } + + pub fn process_weak_stuff( + &self, + worker: &mut GCWorker<Ruby>, + _tracer_context: impl ObjectTracerContext<Ruby>, + ) { + worker.add_work( + WorkBucketStage::VMRefClosure, + ProcessNonParallelObjFreeCanadidates {}, + ); + + for index in 0..self.parallel_obj_free_candidates.len() { + worker.add_work( + WorkBucketStage::VMRefClosure, + ProcessParallelObjFreeCandidates { index }, + ); + } + + worker.add_work(WorkBucketStage::VMRefClosure, ProcessWeakReferences); + + worker.add_work(WorkBucketStage::Prepare, UpdateFinalizerObjIdTables); + + let global_tables_count = (crate::upcalls().global_tables_count)(); + let work_packets = (0..global_tables_count) + .map(|i| Box::new(UpdateGlobalTables { idx: i }) as _) + .collect(); + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(work_packets); + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure] + .bulk_add(vec![Box::new(UpdateWbUnprotectedObjectsList) as _]); + } +} + +fn process_obj_free_candidates(obj_free_candidates: &mut Vec<ObjectReference>) { + // Process obj_free + let mut new_candidates = Vec::new(); + + for object in obj_free_candidates.iter().copied() { + if object.is_reachable() { + // Forward and add back to the candidate list. + let new_object = object.forward(); + trace!("Forwarding obj_free candidate: {object} -> {new_object}"); + new_candidates.push(new_object); + } else { + (upcalls().call_obj_free)(object); + } + } + + *obj_free_candidates = new_candidates; +} + +struct ProcessParallelObjFreeCandidates { + index: usize, +} + +impl GCWork<Ruby> for ProcessParallelObjFreeCandidates { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut obj_free_candidates = crate::binding().weak_proc.parallel_obj_free_candidates + [self.index] + .try_lock() + .expect("Lock for parallel_obj_free_candidates should not be held"); + + process_obj_free_candidates(&mut obj_free_candidates); + } +} + +struct ProcessNonParallelObjFreeCanadidates; + +impl GCWork<Ruby> for ProcessNonParallelObjFreeCanadidates { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut obj_free_candidates = crate::binding() + .weak_proc + .non_parallel_obj_free_candidates + .try_lock() + .expect("Lock for non_parallel_obj_free_candidates should not be held"); + + process_obj_free_candidates(&mut obj_free_candidates); + } +} + +struct ProcessWeakReferences; + +impl GCWork<Ruby> for ProcessWeakReferences { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + if crate::mmtk().get_plan().current_gc_may_move_object() { + let gc_tls: &mut GCThreadTLS = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + let visit_object = |_worker, target_object: ObjectReference, _pin| { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(), + "Destination is not an MMTk object" + ); + + target_object + .get_forwarded_object() + .unwrap_or(target_object) + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + self.process_weak_references(true); + }) + } else { + self.process_weak_references(false); + } + } +} + +impl ProcessWeakReferences { + fn process_weak_references(&mut self, moving_gc: bool) { + let mut weak_references = crate::binding() + .weak_proc + .weak_references + .try_lock() + .expect("Mutators should not be holding the lock."); + + weak_references.retain_mut(|object_ptr| { + let object = object_ptr.get_forwarded_object().unwrap_or(*object_ptr); + + if object != *object_ptr { + *object_ptr = object; + } + + if object.is_reachable() { + (upcalls().handle_weak_references)(object, moving_gc); + + true + } else { + false + } + }); + } +} + +trait GlobalTableProcessingWork { + fn process_table(&mut self); + + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + // `hash_foreach_replace` depends on `gb_object_moved_p` which has to have the semantics + // of `trace_object` due to the way it is used in `UPDATE_IF_MOVED`. + let forward_object = |_worker, object: ObjectReference, _pin| { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "{object} is not an MMTk object" + ); + let result = object.forward(); + trace!("Forwarding reference: {object} -> {result}"); + result + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(forward_object, || { + self.process_table(); + }); + } +} + +struct UpdateFinalizerObjIdTables; +impl GlobalTableProcessingWork for UpdateFinalizerObjIdTables { + fn process_table(&mut self) { + (crate::upcalls().update_finalizer_table)(); + } +} +impl GCWork<Ruby> for UpdateFinalizerObjIdTables { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobalTableProcessingWork::do_work(self, worker, mmtk); + } +} + +struct UpdateGlobalTables { + idx: i32, +} +impl GlobalTableProcessingWork for UpdateGlobalTables { + fn process_table(&mut self) { + (crate::upcalls().update_global_tables)( + self.idx, + crate::mmtk().get_plan().current_gc_may_move_object(), + ) + } +} +impl GCWork<Ruby> for UpdateGlobalTables { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobalTableProcessingWork::do_work(self, worker, mmtk); + } +} + +struct UpdateWbUnprotectedObjectsList; + +impl GCWork<Ruby> for UpdateWbUnprotectedObjectsList { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut objects = crate::binding().wb_unprotected_objects.try_lock().expect( + "Someone is holding the lock of wb_unprotected_objects during weak processing phase?", + ); + + let old_objects = std::mem::take(&mut *objects); + + debug!("Updating {} WB-unprotected objects", old_objects.len()); + + for object in old_objects { + if object.is_reachable() { + // Forward and add back to the candidate list. + let new_object = object.forward(); + trace!("Forwarding WB-unprotected object: {object} -> {new_object}"); + objects.insert(new_object); + } else { + trace!("Removing WB-unprotected object from list: {object}"); + } + } + + debug!("Retained {} live WB-unprotected objects.", objects.len()); + } +} + +// Provide a shorthand `object.forward()`. +trait Forwardable { + fn forward(&self) -> Self; +} + +impl Forwardable for ObjectReference { + fn forward(&self) -> Self { + self.get_forwarded_object().unwrap_or(*self) + } +} diff --git a/gc/wbcheck/extconf.rb b/gc/wbcheck/extconf.rb new file mode 100644 index 0000000000..18b32d820d --- /dev/null +++ b/gc/wbcheck/extconf.rb @@ -0,0 +1,3 @@ +require_relative '../extconf_base' + +create_gc_makefile("wbcheck") diff --git a/gc/wbcheck/wbcheck.c b/gc/wbcheck/wbcheck.c new file mode 100644 index 0000000000..a7d4cd6ccf --- /dev/null +++ b/gc/wbcheck/wbcheck.c @@ -0,0 +1,1936 @@ +#include "internal.h" +#include "ruby/ruby.h" +#include "ruby/assert.h" +#include "ruby/atomic.h" +#include "ruby/debug.h" +#include "ruby/internal/core/rbasic.h" +#include "ruby/st.h" +#include "internal/object.h" +#include "internal/array.h" +#include "internal/class.h" + +#include "ruby/thread.h" +#include "gc/gc.h" +#include "gc/gc_impl.h" + +#include <stdbool.h> +#include <stdarg.h> + +// Debug output control +static bool wbcheck_debug_enabled = false; + +// Verification after write barrier control +static bool wbcheck_verify_after_wb_enabled = false; + +// Useless write barrier warning control +static bool wbcheck_warn_useless_wb_enabled = false; + +static void +wbcheck_debug(const char *format, ...) +{ + if (!wbcheck_debug_enabled) return; + + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); +} + +#define WBCHECK_DEBUG(...) do { \ + if (wbcheck_debug_enabled) { \ + wbcheck_debug(__VA_ARGS__); \ + } \ +} while (0) + +static void +wbcheck_debug_obj_info_dump(VALUE obj) +{ + if (!wbcheck_debug_enabled) return; + char buff[0x100]; + fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); +} + +// Forward declaration +static void lock_and_maybe_gc(void *objspace_ptr); +static void force_gc(void *objspace_ptr); + +// Configure wbcheck from environment variables +static void +wbcheck_configure_from_env(void) +{ + // Configure debug output based on environment variable + const char *debug_env = getenv("WBCHECK_DEBUG"); + if (debug_env && (strcmp(debug_env, "1") == 0 || strcmp(debug_env, "true") == 0)) { + wbcheck_debug_enabled = true; + } + + // Configure verification after write barrier based on environment variable + const char *verify_after_wb_env = getenv("WBCHECK_VERIFY_AFTER_WB"); + if (verify_after_wb_env && (strcmp(verify_after_wb_env, "1") == 0 || strcmp(verify_after_wb_env, "true") == 0)) { + wbcheck_verify_after_wb_enabled = true; + } + + // Configure useless write barrier warnings based on environment variable + const char *warn_useless_wb_env = getenv("WBCHECK_WARN_USELESS_WB"); + if (warn_useless_wb_env && (strcmp(warn_useless_wb_env, "1") == 0 || strcmp(warn_useless_wb_env, "true") == 0)) { + wbcheck_warn_useless_wb_enabled = true; + } +} + +// Define same heap sizes as the default GC +static size_t heap_sizes[] = { + 32, + 40, + 48, + 56, + 64, + 72, + 80, + 96, + 128, + 160, + 256, + 512, + 640, + 768, + 1024, + 0 +}; + +#define HEAP_COUNT ((int)(sizeof(heap_sizes) / sizeof(heap_sizes[0])) - 1) +#define MAX_HEAP_SIZE (heap_sizes[(HEAP_COUNT) - 1]) + +// Object states for verification tracking +typedef enum { + WBCHECK_STATE_CLEAR, // Just allocated or writebarrier_remember, needs reference capture + WBCHECK_STATE_MARKED, // Has valid snapshot, ready for normal operation + WBCHECK_STATE_DIRTY // Has seen writebarrier since last snapshot, queued for verification +} wbcheck_object_state_t; + +// Tri-color marking colors +typedef enum { + WBCHECK_COLOR_WHITE, // Unmarked - will be swept + WBCHECK_COLOR_GRAY, // Marked but children not processed + WBCHECK_COLOR_BLACK // Marked and children processed +} wbcheck_color_t; + +// GC phases +typedef enum { + WBCHECK_PHASE_MUTATOR, // Normal execution + WBCHECK_PHASE_SNAPSHOT, // Collecting references for verification + WBCHECK_PHASE_FULL_GC // Marking objects during full GC +} wbcheck_phase_t; + +// List of objects +typedef struct { + VALUE *items; + size_t count; + size_t capacity; +} wbcheck_object_list_t; + +// Helper functions for object list +static wbcheck_object_list_t * +wbcheck_object_list_init_with_capacity(size_t capacity) +{ + wbcheck_object_list_t *list = calloc(1, sizeof(wbcheck_object_list_t)); + if (!list) rb_bug("wbcheck: failed to allocate object list structure"); + + if (capacity < 4) capacity = 4; + list->items = malloc(capacity * sizeof(VALUE)); + if (!list->items) rb_bug("wbcheck: failed to allocate object list array"); + list->capacity = capacity; + list->count = 0; + return list; +} + +static wbcheck_object_list_t * +wbcheck_object_list_init(void) +{ + return wbcheck_object_list_init_with_capacity(4); +} + +static void +wbcheck_object_list_append(wbcheck_object_list_t *list, VALUE obj) +{ + if (list->count >= list->capacity) { + size_t new_capacity = list->capacity == 0 ? 4 : list->capacity * 2; + VALUE *new_items = realloc(list->items, new_capacity * sizeof(VALUE)); + if (!new_items) rb_bug("wbcheck: failed to reallocate object list array"); + list->items = new_items; + list->capacity = new_capacity; + } + list->items[list->count++] = obj; +} + +static void +wbcheck_object_list_free(wbcheck_object_list_t *list) +{ + if (!list) return; + if (list->items) { + free(list->items); + } + free(list); +} + +static void +wbcheck_object_list_debug_print(wbcheck_object_list_t *list) +{ + if (!wbcheck_debug_enabled) return; + for (size_t i = 0; i < list->count; i++) { + char buff[0x100]; + fprintf(stderr, "-> %s\n", rb_raw_obj_info(buff, sizeof(buff), list->items[i])); + } +} + +static bool +wbcheck_object_list_contains(wbcheck_object_list_t *list, VALUE obj) +{ + for (size_t i = 0; i < list->count; i++) { + if (list->items[i] == obj) { + return true; + } + } + return false; +} + +// Information tracked for each object +typedef struct { + size_t alloc_size; // Allocated size (static) + bool wb_protected; // Write barrier protection status (static) + VALUE finalizers; // Ruby Array of finalizers like [finalizer1, finalizer2, ...] + wbcheck_object_list_t *gc_mark_snapshot; // Snapshot of references from last GC mark + wbcheck_object_list_t *mark_maybe_snapshot; // Conservative refs reported via mark_maybe; needed for liveness, not verifiable + wbcheck_object_list_t *writebarrier_children; // References added via write barriers since last snapshot + wbcheck_object_state_t state; // Current state in verification lifecycle + wbcheck_color_t color; // Tri-color marking color +} rb_wbcheck_object_info_t; + +// Finalizer job types +struct wbcheck_final_job { + struct wbcheck_final_job *next; + enum { + WBCHECK_FINAL_JOB_DFREE, + WBCHECK_FINAL_JOB_FINALIZE, + } kind; + union { + struct { + void (*func)(void *); + void *data; + } dfree; + struct { + VALUE finalizer_array; + } finalize; + } as; +}; + +// wbcheck objspace structure to track all objects +typedef struct { + st_table *object_table; // Hash table to track all allocated objects (VALUE -> rb_wbcheck_object_info_t*) + wbcheck_object_list_t *objects_to_capture; // Objects that need initial reference capture + wbcheck_object_list_t *objects_to_verify; // Objects that need verification after write barriers + wbcheck_object_list_t *current_refs; // Current list for collecting references during marking + wbcheck_object_list_t *current_maybe_refs; // Current list for collecting mark_maybe references during marking + wbcheck_object_list_t *mark_queue; // Queue of gray objects for tri-color marking + wbcheck_object_list_t *weak_references; // Objects holding weak references, found during marking + wbcheck_phase_t phase; // Current GC phase + bool gc_enabled; // Whether GC is allowed to run + bool gc_stress; // GC stress mode (run GC on every allocation) + size_t gc_threshold; // Trigger GC when object count reaches this + size_t missed_write_barrier_parents; // Number of parent objects with missed write barriers + size_t missed_write_barrier_children; // Total number of missed write barriers detected + size_t simulated_gc_count; // Simulated GC count incremented on each GC.start + bool measure_total_time; // Whether to accumulate :time in stats + struct wbcheck_final_job *finalizer_jobs; // Linked list of finalizer jobs + rb_nativethread_lock_t finalizer_lock; // Protects finalizer_jobs list + rb_postponed_job_handle_t finalizer_postponed_job; // Postponed job handle for finalizers +} rb_wbcheck_objspace_t; + +// Global objspace pointer for accessing from obj_slot_size function +static rb_wbcheck_objspace_t *wbcheck_global_objspace = NULL; + +// Forward declarations +static void wbcheck_foreach_object(rb_wbcheck_objspace_t *objspace, int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data), void *data); +static int wbcheck_verify_all_references_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data); +static int wbcheck_update_all_snapshots_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data); +static void wbcheck_run_finalizers_for_object(VALUE obj, rb_wbcheck_object_info_t *info); +static void gc_run_finalizers(void *data); +static void make_final_job(rb_wbcheck_objspace_t *objspace, VALUE obj, VALUE finalizer_array); + +// Helper functions for object tracking +static rb_wbcheck_object_info_t * +wbcheck_get_object_info(VALUE obj) +{ + // Objspace must be initialized by this point + GC_ASSERT(wbcheck_global_objspace); + + st_data_t value; + if (st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value)) { + return (rb_wbcheck_object_info_t *)value; + } + + fprintf(stderr, "wbcheck: object not found in tracking table\n"); + char buff[0x100]; + fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); + + // Force ASAN crash? + ((volatile VALUE *)obj)[0]; + + // Object not found in tracking table - this should never happen + rb_bug("wbcheck: object not found in tracking table"); +} + +static void +wbcheck_report_error(void *objspace_ptr, VALUE parent_obj, wbcheck_object_list_t *current_refs, wbcheck_object_list_t *gc_mark_snapshot, wbcheck_object_list_t *writebarrier_children, wbcheck_object_list_t *missed_refs) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + rb_wbcheck_object_info_t *parent_info = wbcheck_get_object_info(parent_obj); + + size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0; + size_t wb_count = writebarrier_children ? writebarrier_children->count : 0; + + fprintf(stderr, "WBCHECK ERROR: Missed write barrier detected!\n"); + fprintf(stderr, " Parent object: %p (wb_protected: %s)\n", + (void *)parent_obj, parent_info->wb_protected ? "true" : "false"); + char buff[0x100]; + fprintf(stderr, " %s\n", rb_raw_obj_info(buff, sizeof(buff), parent_obj)); + fprintf(stderr, " Reference counts - snapshot: %zu, writebarrier: %zu, current: %zu, missed: %zu\n", + snapshot_count, wb_count, current_refs->count, missed_refs->count); + + for (size_t i = 0; i < missed_refs->count; i++) { + VALUE missed_ref = missed_refs->items[i]; + char buff[0x100]; + fprintf(stderr, " Missing reference to: %p\n %s\n", (void *)missed_ref, rb_raw_obj_info(buff, sizeof(buff), missed_ref)); + } + + fprintf(stderr, "\n"); + objspace->missed_write_barrier_parents++; + objspace->missed_write_barrier_children += missed_refs->count; +} + +static void +wbcheck_compare_references(void *objspace_ptr, VALUE parent_obj, wbcheck_object_list_t *current_refs, wbcheck_object_list_t *gc_mark_snapshot, wbcheck_object_list_t *writebarrier_children) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + (void)objspace; + + size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0; + size_t wb_count = writebarrier_children ? writebarrier_children->count : 0; + + WBCHECK_DEBUG("wbcheck: comparing references for object %p\n", (void *)parent_obj); + WBCHECK_DEBUG("wbcheck: current refs: %zu, snapshot refs: %zu, wb refs: %zu\n", + current_refs->count, snapshot_count, wb_count); + + // Collect missed references (lazily allocated) + wbcheck_object_list_t *missed_refs = NULL; + + // Use circular comparison for better performance when lists are mostly similar + size_t snapshot_idx = 0; + + // Check each object in current_refs to see if it's in either stored list + for (size_t i = 0; i < current_refs->count; i++) { + VALUE current_ref = current_refs->items[i]; + + // Usually the lists are nearly identical. We take advantage of this by + // attempting to loop over both lists in sequence. When the next element + // of the snapshot doesn't match the next element of our current_refs, + // we'll loop around the list to try to find it and continue from that + // match, so any runs of identical items can be matched efficiently. + // + // Pathologically this is O(N**2), but is O(N * num_changes) + bool found_in_snapshot = false; + if (gc_mark_snapshot && snapshot_count > 0) { + size_t start_idx = snapshot_idx; + do { + if (gc_mark_snapshot->items[snapshot_idx] == current_ref) { + found_in_snapshot = true; + snapshot_idx++; + if (snapshot_idx >= snapshot_count) snapshot_idx = 0; + break; + } + snapshot_idx++; + if (snapshot_idx >= snapshot_count) snapshot_idx = 0; + } while (snapshot_idx != start_idx); + } + + if (found_in_snapshot) { + continue; + } + + // Built-in immortal classes can be assigned via RBASIC_SET_CLASS_RAW, + // which bypasses the write barrier. They're pinned as VM roots and + // can never be collected, so a missing WB to them is harmless. + if (RB_TYPE_P(current_ref, T_CLASS) && FL_TEST_RAW(current_ref, RCLASS_IS_ROOT)) { + continue; + } + + // Self reference... Weird but okay I guess + if (current_ref == parent_obj) { + continue; + } + + + // Check if reference exists in writebarrier_children + if (writebarrier_children && wbcheck_object_list_contains(writebarrier_children, current_ref)) { + continue; + } + + // If we get here, the reference wasn't found in either list + // Lazily allocate missed_refs list on first miss + if (!missed_refs) { + missed_refs = wbcheck_object_list_init(); + } + wbcheck_object_list_append(missed_refs, current_ref); + } + + // Report any errors found + if (missed_refs) { + wbcheck_report_error(objspace_ptr, parent_obj, current_refs, gc_mark_snapshot, writebarrier_children, missed_refs); + wbcheck_object_list_free(missed_refs); + } +} + +static void +wbcheck_register_object(void *objspace_ptr, VALUE obj, size_t alloc_size, bool wb_protected) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + // Allocate and initialize object info structure + rb_wbcheck_object_info_t *info = calloc(1, sizeof(rb_wbcheck_object_info_t)); + if (!info) rb_bug("wbcheck_register_object: failed to allocate object info"); + + info->alloc_size = alloc_size; + info->wb_protected = wb_protected; + info->finalizers = 0; /* No finalizers initially */ + info->gc_mark_snapshot = NULL; /* No snapshot initially */ + info->mark_maybe_snapshot = NULL; /* No mark_maybe snapshot initially */ + info->writebarrier_children = NULL; /* No write barrier children initially */ + info->state = WBCHECK_STATE_CLEAR; /* Start in clear state */ + info->color = WBCHECK_COLOR_BLACK; /* Start as black to survive current GC */ + + // Store object info in hash table (VALUE -> rb_wbcheck_object_info_t*) + st_insert(objspace->object_table, (st_data_t)obj, (st_data_t)info); +} + +static void +wbcheck_unregister_object(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + rb_wbcheck_object_info_t *info; + + if (st_delete(objspace->object_table, (st_data_t *)&obj, (st_data_t *)&info)) { + // Free object lists if they were allocated + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->mark_maybe_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + free(info); + } else { + rb_bug("wbcheck_unregister_object: object not found in table"); + } +} + +// Bootup +void * +rb_gc_impl_objspace_alloc(void) +{ + wbcheck_configure_from_env(); + + rb_wbcheck_objspace_t *objspace = calloc(1, sizeof(rb_wbcheck_objspace_t)); + if (!objspace) rb_bug("wbcheck: failed to allocate objspace"); + + objspace->object_table = st_init_numtable(); + if (!objspace->object_table) { + free(objspace); + rb_bug("wbcheck: failed to create object table"); + } + + objspace->objects_to_capture = wbcheck_object_list_init(); // Initialize empty list + objspace->objects_to_verify = wbcheck_object_list_init(); // Initialize empty list + objspace->current_refs = NULL; // No current refs initially + objspace->current_maybe_refs = NULL; // No current maybe refs initially + objspace->mark_queue = wbcheck_object_list_init(); // Initialize mark queue + objspace->weak_references = wbcheck_object_list_init(); // Initialize weak references array + objspace->phase = WBCHECK_PHASE_MUTATOR; // Start in mutator phase + objspace->gc_enabled = true; // GC enabled by default (like default GC) + objspace->gc_stress = false; // GC stress disabled by default + objspace->gc_threshold = 1000; // Start with 1000 objects, will adjust after first GC + objspace->missed_write_barrier_parents = 0; // No errors found yet + objspace->missed_write_barrier_children = 0; // No errors found yet + objspace->simulated_gc_count = 0; // Start with GC count of 0 + objspace->measure_total_time = true; // On by default + + return objspace; +} + +void +rb_gc_impl_objspace_init(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + // Object table is already initialized in objspace_alloc + // Set up global objspace pointer for obj_slot_size function + wbcheck_global_objspace = objspace; + + // Initialize postponed job for finalizers + rb_native_mutex_initialize(&objspace->finalizer_lock); + objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace); +} + +void * +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) +{ + // Stub implementation + return NULL; +} + +void +rb_gc_impl_set_params(void *objspace_ptr) +{ + // Stub implementation +} + +static VALUE +gc_verify_internal_consistency(VALUE self) +{ + return Qnil; +} + +void +rb_gc_impl_init(void) +{ + VALUE gc_constants = rb_hash_new(); + //rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MAX_HEAP_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(3)); + OBJ_FREEZE(gc_constants); + rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants); + + // no-ops for compatibility + rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0); + + rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); + // Stub implementation +} + +size_t * +rb_gc_impl_heap_sizes(void *objspace_ptr) +{ + return heap_sizes; +} + +// Shutdown +void +rb_gc_impl_shutdown_free_objects(void *objspace_ptr) +{ + // Stub implementation +} + +void +rb_gc_impl_objspace_free(void *objspace_ptr) +{ + // This should free everything, but we'll just let it leak +} + +void +rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache) +{ + // Stub implementation +} + +// GC +void +rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + if (objspace) { + objspace->simulated_gc_count++; + } + + if (!ruby_native_thread_p()) return; + + unsigned int lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + force_gc(objspace_ptr); + RB_GC_VM_UNLOCK(lev); +} + +bool +rb_gc_impl_during_gc_p(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->phase != WBCHECK_PHASE_MUTATOR; +} + +static void +wbcheck_prepare_heap_i(VALUE obj, void *data) +{ + rb_gc_prepare_heap_process_object(obj); +} + +void +rb_gc_impl_prepare_heap(void *objspace_ptr) +{ + rb_gc_impl_each_object(objspace_ptr, wbcheck_prepare_heap_i, NULL); +} + +void +rb_gc_impl_gc_enable(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->gc_enabled = true; +} + +void +rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->gc_enabled = false; +} + +bool +rb_gc_impl_gc_enabled_p(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->gc_enabled; +} + +void +rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->gc_stress = RTEST(flag); +} + +VALUE +rb_gc_impl_stress_get(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->gc_stress ? Qtrue : Qfalse; +} + +VALUE +rb_gc_impl_config_get(void *objspace_ptr) +{ + return rb_hash_new(); +} + +void +rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) +{ +} + +static wbcheck_object_list_t * +wbcheck_collect_references_from_object(VALUE obj, rb_wbcheck_object_info_t *info) +{ + rb_wbcheck_objspace_t *objspace = wbcheck_global_objspace; + + // Use combination of writebarrier children and last snapshot as capacity hint + size_t snapshot_count = (info->gc_mark_snapshot) ? info->gc_mark_snapshot->count : 0; + size_t wb_children_count = (info->writebarrier_children) ? info->writebarrier_children->count : 0; + size_t capacity_hint = snapshot_count + wb_children_count; + wbcheck_object_list_t *new_list = wbcheck_object_list_init_with_capacity(capacity_hint); + + // Set up objspace state for marking. current_maybe_refs is allocated lazily + // by rb_gc_impl_mark_maybe, since most objects have no conservative refs. + objspace->current_refs = new_list; + objspace->current_maybe_refs = NULL; + objspace->phase = WBCHECK_PHASE_SNAPSHOT; + + // Use the marking infrastructure to collect references + rb_gc_mark_children(objspace, obj); + + // Clean up objspace state + objspace->phase = WBCHECK_PHASE_MUTATOR; + objspace->current_refs = NULL; + + // Update the mark_maybe snapshot in place. These references don't participate + // in verification, but we need to keep them so full GC can mark them gray. + wbcheck_object_list_free(info->mark_maybe_snapshot); + info->mark_maybe_snapshot = objspace->current_maybe_refs; + objspace->current_maybe_refs = NULL; + + if (wbcheck_debug_enabled) { + WBCHECK_DEBUG("wbcheck: collected %zu references from %p\n", new_list->count, (void *)obj); + char buff[0x100]; + fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); + wbcheck_object_list_debug_print(new_list); + } + + return new_list; +} + +static void +wbcheck_collect_initial_references(void *objspace_ptr, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: collecting initial references from %p:\n", obj); + wbcheck_debug_obj_info_dump(obj); + + // Get the object info and set the initial GC mark snapshot + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + wbcheck_object_list_t *new_list = wbcheck_collect_references_from_object(obj, info); + RUBY_ASSERT(!info->gc_mark_snapshot); + RUBY_ASSERT(info->state == WBCHECK_STATE_CLEAR); + info->gc_mark_snapshot = new_list; // Set the initial snapshot + info->state = WBCHECK_STATE_MARKED; // Transition to marked state +} + +static void +wbcheck_verify_object_references(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + // Ignore objects which are not write barrier protected + if (!info->wb_protected) { + return; + } + + // We hadn't captured initial references + if (info->state == WBCHECK_STATE_CLEAR) { + RUBY_ASSERT(!info->gc_mark_snapshot); + return; + } + + WBCHECK_DEBUG("wbcheck: verifying references for object:\n"); + wbcheck_debug_obj_info_dump(obj); + + // Get the current references from the object + wbcheck_object_list_t *current_refs = wbcheck_collect_references_from_object(obj, info); + + // Check for useless write barriers before clearing them + if (wbcheck_warn_useless_wb_enabled && info->writebarrier_children) { + for (size_t i = 0; i < info->writebarrier_children->count; i++) { + VALUE wb_ref = info->writebarrier_children->items[i]; + if (!wbcheck_object_list_contains(current_refs, wb_ref)) { + fprintf(stderr, "WBCHECK WARNING: Potentially useless write barrier detected for object %p\n", (void *)obj); + fprintf(stderr, " Write barrier was recorded for reference to %p, but object no longer references it\n", (void *)wb_ref); + char buff[0x100]; + fprintf(stderr, " Parent: %s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); + fprintf(stderr, " Stale reference: %s\n", rb_raw_obj_info(buff, sizeof(buff), wb_ref)); + } + } + } + + // Compare current_refs against both stored lists to detect missed write barriers + wbcheck_compare_references(objspace_ptr, obj, current_refs, info->gc_mark_snapshot, info->writebarrier_children); + + // Update the snapshot with current references and clear write barrier children + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + info->gc_mark_snapshot = current_refs; + info->writebarrier_children = NULL; + info->state = WBCHECK_STATE_MARKED; // Back to marked state after verification +} + +// Mark object as gray (add to mark queue) +static void +wbcheck_mark_gray(rb_wbcheck_objspace_t *objspace, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + st_data_t value; + if (!st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_bug("wbcheck: asked to mark object %p not in our object table", (void *)obj); + } + + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + if (info->color != WBCHECK_COLOR_WHITE) { + return; // Already marked + } + + info->color = WBCHECK_COLOR_GRAY; + wbcheck_object_list_append(objspace->mark_queue, obj); + + if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) { + wbcheck_object_list_append(objspace->weak_references, obj); + } + + WBCHECK_DEBUG("wbcheck: marked gray: %p\n", (void *)obj); +} + +// Reset all objects to white +static int +st_foreach_reset_white(st_data_t key, st_data_t val, st_data_t arg) +{ + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + info->color = WBCHECK_COLOR_WHITE; + return ST_CONTINUE; +} + +// Mark all finalizer arrays to keep them alive during GC +static int +st_foreach_mark_finalizers(st_data_t key, st_data_t val, st_data_t arg) +{ + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)arg; + + if (info->finalizers) { + wbcheck_mark_gray(objspace, info->finalizers); + } + + return ST_CONTINUE; +} + +// Full mark phase using tri-color marking with snapshots +static void +wbcheck_mark_phase(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: starting GC mark phase\n"); + + objspace->phase = WBCHECK_PHASE_FULL_GC; + + // Clear mark queue and reset all objects to white + objspace->mark_queue->count = 0; + st_foreach(objspace->object_table, st_foreach_reset_white, 0); + + // Mark all finalizer arrays first to keep them alive + st_foreach(objspace->object_table, st_foreach_mark_finalizers, (st_data_t)objspace); + + // Mark finalizer arrays in pending jobs to keep them alive. + // No lock needed: all other threads are stopped during GC. + struct wbcheck_final_job *job = objspace->finalizer_jobs; + while (job != NULL) { + switch (job->kind) { + case WBCHECK_FINAL_JOB_DFREE: + break; + case WBCHECK_FINAL_JOB_FINALIZE: + wbcheck_mark_gray(objspace, job->as.finalize.finalizer_array); + break; + default: + rb_bug("wbcheck_mark_phase: unknown final job type %d", job->kind); + } + job = job->next; + } + + // Mark roots gray + rb_gc_save_machine_context(); + rb_gc_mark_roots(objspace, NULL); + + // Process gray queue until empty + while (objspace->mark_queue->count > 0) { + // Get last object from queue (LIFO) + VALUE obj = objspace->mark_queue->items[--objspace->mark_queue->count]; + + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + if (info->color == WBCHECK_COLOR_GRAY) { + // Mark all children from snapshot gray + if (info->gc_mark_snapshot) { + for (size_t i = 0; i < info->gc_mark_snapshot->count; i++) { + wbcheck_mark_gray(objspace, info->gc_mark_snapshot->items[i]); + } + } + + // Conservatively-scanned children must also be kept alive + if (info->mark_maybe_snapshot) { + for (size_t i = 0; i < info->mark_maybe_snapshot->count; i++) { + wbcheck_mark_gray(objspace, info->mark_maybe_snapshot->items[i]); + } + } + + // Mark this object black + info->color = WBCHECK_COLOR_BLACK; + WBCHECK_DEBUG("wbcheck: marked black: %p\n", (void *)obj); + } + } + } + + objspace->phase = WBCHECK_PHASE_MUTATOR; + + WBCHECK_DEBUG("wbcheck: tri-color mark phase complete\n"); +} + +// Sweep phase callback - free white objects +static int +wbcheck_sweep_callback(st_data_t key, st_data_t val, st_data_t arg, int error) +{ + VALUE obj = (VALUE)key; + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)arg; + + if (info->color == WBCHECK_COLOR_WHITE) { + WBCHECK_DEBUG("wbcheck: sweeping unmarked object %p\n", (void *)obj); + + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); + + // Clear weak references first + rb_gc_obj_free_vm_weak_references(obj); + + // Queue finalizers for postponed job if they exist + if (info->finalizers) { + make_final_job(objspace, obj, info->finalizers); + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } + + // Call rb_gc_obj_free which handles finalizers/zombies + if (rb_gc_obj_free(objspace, obj)) { + // Object was actually freed, clean up our tracking + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->mark_maybe_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + free(info); + + // Free the actual object memory + free((void *)obj); + + return ST_DELETE; // Remove from hash table + } else { + // Object became a zombie - it will be freed by postponed job + // Remove from tracking since we can't safely access it anymore + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->mark_maybe_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + free(info); + + // Free the actual object memory + free((void *)obj); + + return ST_DELETE; // Remove from hash table + } + } + + return ST_CONTINUE; // Keep marked objects +} + +static void +wbcheck_sweep_phase(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: starting sweep phase\n"); + + size_t objects_before = st_table_size(objspace->object_table); + + // Sweep unmarked objects + st_foreach_check(objspace->object_table, wbcheck_sweep_callback, (st_data_t)objspace, 0); + + size_t objects_after = st_table_size(objspace->object_table); + size_t freed_objects = objects_before - objects_after; + + // Update GC threshold: 2x the live set after GC + objspace->gc_threshold = objects_after * 2; + + WBCHECK_DEBUG("wbcheck: sweep phase complete - freed %zu objects (%zu -> %zu), new threshold: %zu\n", + freed_objects, objects_before, objects_after, objspace->gc_threshold); +} + +// Process weak references after marking - call rb_gc_handle_weak_references +// on each object that was flagged with RUBY_FL_WEAK_REFERENCE and collected +// during the mark phase. +static void +wbcheck_process_weak_references(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: processing %zu weak reference objects\n", objspace->weak_references->count); + + for (size_t i = 0; i < objspace->weak_references->count; i++) { + VALUE obj = objspace->weak_references->items[i]; + rb_gc_handle_weak_references(obj); + } + + objspace->weak_references->count = 0; +} + +// Full GC: verify all objects then mark from roots +static void +wbcheck_full_gc(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: starting full GC\n"); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_ENTER); + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START); + + // First, update snapshots for all objects (verify wb_protected ones) + WBCHECK_DEBUG("wbcheck: updating snapshots for all objects\n"); + wbcheck_foreach_object(objspace, wbcheck_update_all_snapshots_callback, objspace); + + // Now start tri-color marking + wbcheck_mark_phase(objspace); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_MARK); + + // Process weak references after marking, before sweeping + wbcheck_process_weak_references(objspace); + + // Sweep unmarked objects + wbcheck_sweep_phase(objspace); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP); + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_EXIT); + + WBCHECK_DEBUG("wbcheck: full GC complete\n"); +} + +static void +gc_step(void *objspace_ptr, bool force) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + // Not initialized yet + if (!objspace) return; + + if (!objspace->gc_enabled && !force) return; + + // Process all objects that need verification after write barriers (if enabled) + if (wbcheck_verify_after_wb_enabled) { + for (size_t i = 0; i < objspace->objects_to_verify->count; i++) { + VALUE obj = objspace->objects_to_verify->items[i]; + wbcheck_verify_object_references(objspace_ptr, obj); + } + + // Clear the list after processing + objspace->objects_to_verify->count = 0; + + // If any new errors were detected during verification, exit immediately + if (objspace->missed_write_barrier_parents > 0) { + rb_bug("wbcheck: missed write barrier detected during immediate verification (WBCHECK_VERIFY_AFTER_WB=1)"); + } + } + + // Process all objects that need initial reference capture + for (size_t i = 0; i < objspace->objects_to_capture->count; i++) { + VALUE obj = objspace->objects_to_capture->items[i]; + wbcheck_collect_initial_references(objspace_ptr, obj); + } + + // Clear the list after processing + objspace->objects_to_capture->count = 0; + + // Run full GC if forced, if we exceed the threshold, or if gc_stress is enabled + if (ruby_native_thread_p() && + (force || + (objspace->gc_enabled && + (objspace->gc_stress || st_table_size(objspace->object_table) >= objspace->gc_threshold)))) { + wbcheck_full_gc(objspace); + } + +} + +static void +maybe_gc(void *objspace_ptr) +{ + gc_step(objspace_ptr, false); +} + +static void +force_gc(void *objspace_ptr) +{ + gc_step(objspace_ptr, true); +} + +int ruby_thread_has_gvl_p(void); + +static void * +lock_and_maybe_gc_gvl(void *objspace_ptr) +{ + unsigned int lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + maybe_gc(objspace_ptr); + + RB_GC_VM_UNLOCK(lev); + return NULL; +} + +static void +lock_and_maybe_gc(void *objspace_ptr) +{ + if (!ruby_native_thread_p()) return; + + if (!ruby_thread_has_gvl_p()) { + rb_thread_call_with_gvl(lock_and_maybe_gc_gvl, objspace_ptr); + } + else { + lock_and_maybe_gc_gvl(objspace_ptr); + } +} + +VALUE +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) +{ + unsigned int lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + // Check if we should trigger GC before allocating + maybe_gc(objspace_ptr); + + // Ensure minimum allocation size of BASE_SLOT_SIZE + alloc_size = heap_sizes[rb_gc_impl_heap_id_for_size(objspace_ptr, alloc_size)]; + + // Allocate memory for the object + VALUE *mem = malloc(alloc_size); + if (!mem) rb_bug("FIXME: malloc failed"); + + // Initialize the object + VALUE obj = (VALUE)mem; + RBASIC(obj)->flags = flags; + *((VALUE *)&RBASIC(obj)->klass) = klass; + + // Register the new object in our tracking table + wbcheck_register_object(objspace_ptr, obj, alloc_size, wb_protected); + + // Add this object to the list of objects that need initial reference capture + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + wbcheck_object_list_append(objspace->objects_to_capture, obj); + + RB_GC_VM_UNLOCK(lev); + return obj; +} + +size_t +rb_gc_impl_obj_slot_size(VALUE obj) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + size_t result = info->alloc_size; + + RB_GC_VM_UNLOCK(lev); + return result; +} + +size_t +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + if (size <= heap_sizes[i]) return i; + } + rb_bug("size too big"); +} + +bool +rb_gc_impl_size_allocatable_p(size_t size) +{ + // Only allow sizes up to the largest heap size + return size <= MAX_HEAP_SIZE; +} + +// Malloc +void * +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + if (gc_allowed) { + lock_and_maybe_gc(objspace_ptr); + } + return malloc(size); +} + +void * +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + if (gc_allowed) { + lock_and_maybe_gc(objspace_ptr); + } + return calloc(1, size); +} + +void * +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) +{ + if (gc_allowed) { + lock_and_maybe_gc(objspace_ptr); + } + return realloc(ptr, new_size); +} + +void +rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) +{ + free(ptr); +} + +void +rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) +{ + // For wbcheck, we don't track memory usage +} + +// Marking +static void +gc_mark(rb_wbcheck_objspace_t *objspace, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: gc_mark called\n"); + wbcheck_debug_obj_info_dump(obj); + + if (RB_SPECIAL_CONST_P(obj)) return; + + switch (objspace->phase) { + case WBCHECK_PHASE_SNAPSHOT: + // Collecting references during verification + GC_ASSERT(objspace->current_refs); + wbcheck_object_list_append(objspace->current_refs, obj); + break; + case WBCHECK_PHASE_FULL_GC: + // Marking during full GC + wbcheck_mark_gray(objspace, obj); + break; + case WBCHECK_PHASE_MUTATOR: + // Should not be called during mutator phase + rb_bug("wbcheck: gc_mark called during mutator phase"); + break; + } +} + +void +rb_gc_impl_mark(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + gc_mark(objspace, obj); +} + +void +rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + gc_mark(objspace, *ptr); +} + +void +rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + gc_mark(objspace, obj); +} + +void +rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)obj)) return; + + switch (objspace->phase) { + case WBCHECK_PHASE_SNAPSHOT: + // We don't know if this is actually a reference or just a value + // that looks like one, so we can't expect a write barrier for it. + // Keep it separate from the verifiable refs, but retain it so full + // GC can mark the target gray if it does turn out to be live. + if (!objspace->current_maybe_refs) { + objspace->current_maybe_refs = wbcheck_object_list_init(); + } + wbcheck_object_list_append(objspace->current_maybe_refs, obj); + break; + case WBCHECK_PHASE_FULL_GC: + wbcheck_mark_gray(objspace, obj); + break; + case WBCHECK_PHASE_MUTATOR: + rb_bug("wbcheck: rb_gc_impl_mark_maybe called during mutator phase"); + break; + } +} + +// Weak references +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) +{ + FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE); +} + +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + return info->color != WBCHECK_COLOR_WHITE; + } + + return false; +} + +// Compaction +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + /* no-op */ +} + +bool +rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) +{ + // Stub implementation + return false; +} + +VALUE +rb_gc_impl_location(void *objspace_ptr, VALUE value) +{ + // Stub implementation + return Qnil; +} + +// Write barriers +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + if (RB_SPECIAL_CONST_P(b)) return; + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + // Get the object info for the parent object (a) + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(a); + + // Only record the write barrier if we have a valid snapshot + if (info->state != WBCHECK_STATE_CLEAR) { + RUBY_ASSERT(info->gc_mark_snapshot); + + // Initialize writebarrier_children list if it doesn't exist + if (!info->writebarrier_children) { + info->writebarrier_children = wbcheck_object_list_init(); + } + + // Add the new reference to the write barrier children list + wbcheck_object_list_append(info->writebarrier_children, b); + + WBCHECK_DEBUG("wbcheck: write barrier recorded reference from %p to %p\n", (void *)a, (void *)b); + + // If verification after write barrier is enabled, queue the object for verification + if (wbcheck_verify_after_wb_enabled && info->state != WBCHECK_STATE_DIRTY) { + WBCHECK_DEBUG("wbcheck: queueing object for verification after write barrier\n"); + info->state = WBCHECK_STATE_DIRTY; // Mark as dirty + wbcheck_object_list_append(objspace->objects_to_verify, a); + } + } else { + WBCHECK_DEBUG("wbcheck: write barrier skipped (snapshot not initialized) from %p to %p\n", (void *)a, (void *)b); + } + + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: writebarrier_unprotect called on object %p\n", (void *)obj); + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + info->wb_protected = false; + + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: writebarrier_remember called on object %p\n", (void *)obj); + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + // Clear existing references since they may be stale + if (info->state != WBCHECK_STATE_CLEAR) { + RUBY_ASSERT(info->gc_mark_snapshot); + wbcheck_object_list_free(info->gc_mark_snapshot); + info->gc_mark_snapshot = NULL; + + wbcheck_object_list_free(info->mark_maybe_snapshot); + info->mark_maybe_snapshot = NULL; + + // Only re-add to objects_to_capture if it had previous snapshot + // (new objects don't need to be re-added since they'll be captured at allocation) + wbcheck_object_list_append(objspace->objects_to_capture, obj); + + // Also clear write barrier children + if (info->writebarrier_children) { + wbcheck_object_list_free(info->writebarrier_children); + info->writebarrier_children = NULL; + } + + // Reset to clear state + info->state = WBCHECK_STATE_CLEAR; + } + RUBY_ASSERT(!info->gc_mark_snapshot); + RUBY_ASSERT(!info->mark_maybe_snapshot); + RUBY_ASSERT(!info->writebarrier_children); + + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +// Heap walking +struct wbcheck_foreach_data { + int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data); + void *data; +}; + +static int +wbcheck_foreach_object_i(st_data_t key, st_data_t val, st_data_t arg) +{ + VALUE obj = (VALUE)key; + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + struct wbcheck_foreach_data *foreach_data = (struct wbcheck_foreach_data *)arg; + + return foreach_data->callback(obj, info, foreach_data->data); +} + +static void +wbcheck_foreach_object(rb_wbcheck_objspace_t *objspace, int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data), void *data) +{ + struct wbcheck_foreach_data foreach_data = { + .callback = callback, + .data = data + }; + + st_foreach(objspace->object_table, wbcheck_foreach_object_i, (st_data_t)&foreach_data); +} + +// Helper to collect all objects into a snapshot list +static int +wbcheck_snapshot_collector(st_data_t key, st_data_t val, st_data_t arg) +{ + VALUE obj = (VALUE)key; + wbcheck_object_list_t *snapshot = (wbcheck_object_list_t *)arg; + wbcheck_object_list_append(snapshot, obj); + return ST_CONTINUE; +} + +// Take a snapshot of all objects for safe iteration +static wbcheck_object_list_t * +wbcheck_take_object_snapshot(rb_wbcheck_objspace_t *objspace) +{ + size_t object_count = st_table_size(objspace->object_table); + wbcheck_object_list_t *snapshot = wbcheck_object_list_init_with_capacity(object_count); + st_foreach(objspace->object_table, wbcheck_snapshot_collector, (st_data_t)snapshot); + return snapshot; +} + + +void +rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + wbcheck_object_list_t *snapshot = wbcheck_take_object_snapshot(objspace); + + for (size_t i = 0; i < snapshot->count; i++) { + VALUE obj = snapshot->items[i]; + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + int result = callback( + (void *)obj, + (void *)((char *)obj + info->alloc_size), + info->alloc_size, + data + ); + if (result != 0) break; + } + } + + wbcheck_object_list_free(snapshot); +} + +void +rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + wbcheck_object_list_t *snapshot = wbcheck_take_object_snapshot(objspace); + + for (size_t i = 0; i < snapshot->count; i++) { + VALUE obj = snapshot->items[i]; + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + func(obj, data); + } + } + + wbcheck_object_list_free(snapshot); +} + +static void +finalizer_jobs_push(rb_wbcheck_objspace_t *objspace, struct wbcheck_final_job *job) +{ + rb_native_mutex_lock(&objspace->finalizer_lock); + job->next = objspace->finalizer_jobs; + objspace->finalizer_jobs = job; + rb_native_mutex_unlock(&objspace->finalizer_lock); +} + +static struct wbcheck_final_job * +finalizer_jobs_pop(rb_wbcheck_objspace_t *objspace) +{ + rb_native_mutex_lock(&objspace->finalizer_lock); + struct wbcheck_final_job *job = objspace->finalizer_jobs; + if (job) { + objspace->finalizer_jobs = job->next; + } + rb_native_mutex_unlock(&objspace->finalizer_lock); + return job; +} + +// Finalizers +void +rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) +{ + if (dfree == NULL) return; + + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + struct wbcheck_final_job *job = malloc(sizeof(struct wbcheck_final_job)); + job->kind = WBCHECK_FINAL_JOB_DFREE; + job->as.dfree.func = dfree; + job->as.dfree.data = data; + + finalizer_jobs_push(objspace, job); + + if (!ruby_free_at_exit_p()) { + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } + + WBCHECK_DEBUG("wbcheck: made zombie for object %p with dfree function\n", (void *)obj); +} + +VALUE +rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + (void)objspace_ptr; + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + GC_ASSERT(!OBJ_FROZEN(obj)); + + RBASIC(obj)->flags |= FL_FINALIZE; + + VALUE table = info->finalizers; + VALUE result = block; + + if (!table) { + /* First finalizer for this object - store object ID as first element */ + table = rb_ary_new3(2, rb_obj_id(obj), block); + rb_obj_hide(table); + info->finalizers = table; + } else { + /* Check for duplicate finalizers (skip index 0 which is object ID) */ + long len = RARRAY_LEN(table); + long i; + + for (i = 1; i < len; i++) { + VALUE recv = RARRAY_AREF(table, i); + if (rb_equal(recv, block)) { + result = recv; /* Duplicate found, return existing */ + goto unlock_and_return; + } + } + + rb_ary_push(table, block); + } + +unlock_and_return: + RB_GC_VM_UNLOCK(lev); + return result; +} + +void +rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + (void)objspace_ptr; + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + GC_ASSERT(!OBJ_FROZEN(obj)); + + info->finalizers = 0; + FL_UNSET(obj, FL_FINALIZE); + + RB_GC_VM_UNLOCK(lev); +} + +void +rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) +{ + (void)objspace_ptr; + + if (!FL_TEST(obj, FL_FINALIZE)) return; + + unsigned int lev = RB_GC_VM_LOCK(); + + rb_wbcheck_object_info_t *src_info = wbcheck_get_object_info(obj); + rb_wbcheck_object_info_t *dest_info = wbcheck_get_object_info(dest); + + if (src_info->finalizers) { + VALUE table = rb_ary_dup(src_info->finalizers); + RARRAY_ASET(table, 0, rb_obj_id(dest)); + rb_obj_hide(table); + dest_info->finalizers = table; + FL_SET(dest, FL_FINALIZE); + } + + RB_GC_VM_UNLOCK(lev); +} + +static VALUE +wbcheck_get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i + 1); +} + +static void +make_final_job(rb_wbcheck_objspace_t *objspace, VALUE obj, VALUE finalizer_array) +{ + RUBY_ASSERT(RB_FL_TEST(obj, FL_FINALIZE)); + RUBY_ASSERT(RB_BUILTIN_TYPE(finalizer_array) == T_ARRAY); + + RB_FL_UNSET(obj, FL_FINALIZE); + + struct wbcheck_final_job *job = malloc(sizeof(struct wbcheck_final_job)); + job->kind = WBCHECK_FINAL_JOB_FINALIZE; + job->as.finalize.finalizer_array = finalizer_array; + + finalizer_jobs_push(objspace, job); +} + +static void +gc_run_finalizers(void *data) +{ + rb_wbcheck_objspace_t *objspace = data; + + rb_gc_set_pending_interrupt(); + + struct wbcheck_final_job *job; + while ((job = finalizer_jobs_pop(objspace)) != NULL) { + switch (job->kind) { + case WBCHECK_FINAL_JOB_DFREE: + job->as.dfree.func(job->as.dfree.data); + break; + case WBCHECK_FINAL_JOB_FINALIZE: { + VALUE finalizer_array = job->as.finalize.finalizer_array; + + rb_gc_run_obj_finalizer( + RARRAY_AREF(finalizer_array, 0), + RARRAY_LEN(finalizer_array) - 1, + wbcheck_get_final, + (void *)finalizer_array + ); + + RB_GC_GUARD(finalizer_array); + break; + } + } + + free(job); + } + + rb_gc_unset_pending_interrupt(); +} + +static void +wbcheck_run_finalizers_for_object(VALUE obj, rb_wbcheck_object_info_t *info) +{ + if (info->finalizers) { + VALUE table = info->finalizers; + long count = RARRAY_LEN(table) - 1; + rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), count, wbcheck_get_final, (void *)table); + FL_UNSET(obj, FL_FINALIZE); + } + info->finalizers = 0; +} + +static int +wbcheck_shutdown_call_finalizer_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + wbcheck_run_finalizers_for_object(obj, info); + return ST_CONTINUE; /* Keep iterating through all objects */ +} + +static int +wbcheck_verify_all_references_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + void *objspace_ptr = data; + wbcheck_verify_object_references(objspace_ptr, obj); + return ST_CONTINUE; +} + +static int +wbcheck_update_all_snapshots_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + void *objspace_ptr = data; + + // For wb_protected objects, do full verification if they have a snapshot + if (info->wb_protected && info->state != WBCHECK_STATE_CLEAR) { + wbcheck_verify_object_references(objspace_ptr, obj); + } else { + // For CLEAR objects (wb_protected or not) and non-wb_protected objects, just take a new snapshot + wbcheck_object_list_t *current_refs = wbcheck_collect_references_from_object(obj, info); + wbcheck_object_list_free(info->gc_mark_snapshot); + info->gc_mark_snapshot = current_refs; + info->state = WBCHECK_STATE_MARKED; + } + + return ST_CONTINUE; +} + +static int +wbcheck_shutdown_finalizer_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + void *objspace_ptr = data; + + if (rb_gc_shutdown_call_finalizer_p(obj)) { + WBCHECK_DEBUG("wbcheck: finalizing object during shutdown: %p\n", (void *)obj); + rb_gc_obj_free_vm_weak_references(obj); + if (rb_gc_obj_free(objspace_ptr, obj)) { + RBASIC(obj)->flags = 0; + } + } + + return ST_CONTINUE; +} + + +void +rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + // Call all finalizers for all objects using our shared iteration helper + wbcheck_foreach_object(objspace, wbcheck_shutdown_call_finalizer_callback, NULL); + + // After all finalizers have been called, verify all object references + unsigned int verify_lev = RB_GC_VM_LOCK(); + WBCHECK_DEBUG("wbcheck: verifying references for all objects after finalizers\n"); + wbcheck_foreach_object(objspace, wbcheck_verify_all_references_callback, objspace_ptr); + WBCHECK_DEBUG("wbcheck: finished verifying all object references\n"); + RB_GC_VM_UNLOCK(verify_lev); + + // Print summary and exit with error code if violations were found + if (objspace->missed_write_barrier_parents > 0 || objspace->missed_write_barrier_children > 0) { + fprintf(stderr, "WBCHECK SUMMARY: Found %zu objects with missed write barriers (%zu total violations)\n", + objspace->missed_write_barrier_parents, objspace->missed_write_barrier_children); + + + exit(1); // Exit with error code to indicate violations were found + } else { + WBCHECK_DEBUG("wbcheck: no write barrier violations detected\n"); + } + + // Call rb_gc_obj_free on objects that need shutdown finalization (File, Data with dfree, etc.) + unsigned int lev = RB_GC_VM_LOCK(); + WBCHECK_DEBUG("wbcheck: calling rb_gc_obj_free on objects that need shutdown finalization\n"); + wbcheck_foreach_object(objspace, wbcheck_shutdown_finalizer_callback, objspace_ptr); + WBCHECK_DEBUG("wbcheck: finished calling rb_gc_obj_free\n"); + + // Run any pending finalizer jobs (dfree functions) + WBCHECK_DEBUG("wbcheck: running pending finalizer jobs\n"); + gc_run_finalizers(objspace); + WBCHECK_DEBUG("wbcheck: finished running finalizer jobs\n"); + RB_GC_VM_UNLOCK(lev); +} + +// Forking +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + // Stub implementation +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + // Stub implementation +} + +// Statistics +void +rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->measure_total_time = RTEST(flag); +} + +bool +rb_gc_impl_get_measure_total_time(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->measure_total_time; +} + +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->measure_total_time ? objspace->simulated_gc_count : 0; +} + +size_t +rb_gc_impl_gc_count(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + if (objspace) { + return objspace->simulated_gc_count; + } + return 0; +} + +VALUE +rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key) +{ + // Stub implementation + return Qnil; +} + +VALUE +rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + VALUE hash = Qnil, key = Qnil; + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + hash = hash_or_sym; + } + else if (SYMBOL_P(hash_or_sym)) { + key = hash_or_sym; + } + else { + rb_bug("non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == ID2SYM(rb_intern(#name))) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, ID2SYM(rb_intern(#name)), SIZET2NUM(attr)); + + /* Pretend each GC takes 1ms; :time is reported in milliseconds. */ + SET(count, objspace->simulated_gc_count); + SET(time, objspace->measure_total_time ? objspace->simulated_gc_count : 0); + SET(tracked_objects, st_table_size(objspace->object_table)); +#undef SET + + if (!NIL_P(key)) { + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); + } + + rb_hash_aset(hash, ID2SYM(rb_intern("gc_implementation")), rb_str_new_cstr("wbcheck")); + + return hash; +} + +VALUE +rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) +{ + if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) { + int heap_idx = FIX2INT(heap_name); + if (heap_idx < 0 || heap_idx >= HEAP_COUNT) { + rb_raise(rb_eArgError, "size pool index out of range"); + } + + if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) { + return SIZET2NUM(heap_sizes[heap_idx]); + } + + return Qundef; + } + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + return hash_or_sym; + } + + return Qundef; +} + +const char * +rb_gc_impl_active_gc_name(void) +{ + // Stub implementation + return "wbcheck"; +} + +// Miscellaneous +#define WBCHECK_OBJECT_METADATA_ENTRY_COUNT 2 +static struct rb_gc_object_metadata_entry object_metadata_entries[WBCHECK_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) +{ + static ID ID_object_id, ID_shareable; + + if (!ID_object_id) { + ID_object_id = rb_intern("object_id"); + ID_shareable = rb_intern("shareable"); + } + + size_t n = 0; + +#define SET_ENTRY(na, v) do { \ + GC_ASSERT(n < WBCHECK_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue); +#undef SET_ENTRY + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; + + return object_metadata_entries; +} + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + GC_ASSERT(wbcheck_global_objspace); + + unsigned int lev = RB_GC_VM_LOCK(); + + // Check if this pointer exists in our object tracking table + st_data_t value; + bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)ptr, &value); + + RB_GC_VM_UNLOCK(lev); + return result; +} + +bool +rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + // Check if this pointer exists in our object tracking table + st_data_t value; + bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value); + + RB_GC_VM_UNLOCK(lev); + return !result; +} + +void +rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) +{ + // Stub implementation +} + +void +rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) +{ + rb_wbcheck_object_info_t *src_info = wbcheck_get_object_info(obj); + + if (!src_info->wb_protected) { + rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest); + } + rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj); +} + |