diff options
Diffstat (limited to 'gc')
31 files changed, 10038 insertions, 1524 deletions
diff --git a/gc/README.md b/gc/README.md new file mode 100644 index 0000000000..cb71357973 --- /dev/null +++ b/gc/README.md @@ -0,0 +1,37 @@ +# Ruby's Garbage Collectors + +This directory contains implementations for Ruby's garbage collector (GC). The GC implementations use the Modular GC API to interact with Ruby. For more details about this API, see the [Modular GC API](#modular-gc-api) section. + +Two GC implementations are included in Ruby: + +- Default: The GC implementation that is used by default in Ruby. This GC is stable and production ready. The implementation uses a mark-sweep-compact algorithm. +- MMTk: An experimental implementation using the [MMTk](https://www.mmtk.io/) framework. The code lives in the [ruby/mmtk](https://github.com/ruby/mmtk) repository and is synchronized here. MMTk provides a [wide variety of GC algorithms](https://www.mmtk.io/status#implemented-collectors) to choose from. For usage instructions and current progress, refer to the [ruby/mmtk](https://github.com/ruby/mmtk) repository. + +## Building guide + +> [!TIP] +> If you are not sure how to build Ruby, follow the [Building Ruby](https://docs.ruby-lang.org/en/master/contributing/building_ruby_md.html) guide. + +> [!IMPORTANT] +> Ruby's modular GC feature is experimental and subject to change. There may be bugs or performance impacts. Use at your own risk. + +### Building Ruby with Modular GC + +1. Configure Ruby with the `--with-modular-gc=<dir>` option, where `dir` is the directory you want to place the built GC libraries into. +2. Build Ruby as usual. + +### Building GC implementations shipped with Ruby + +1. Build your desired GC implementation with `make install-modular-gc MODULAR_GC=<impl>`. This will build the GC implementation and place the built library into the `dir` specified in step 1. `impl` can be one of: + - `default`: The default GC that Ruby ships with. + - `mmtk`: The GC that uses [MMTk](https://www.mmtk.io/) as the back-end. See Ruby-specific details in the [ruby/mmtk](https://github.com/ruby/mmtk) repository. +2. Run your desired GC implementation by setting the `RUBY_GC_LIBRARY=<lib>` environment variable, where `lib` could be `default`, `mmtk`, or your own implementation (as long as you place it in the `dir` specified in step 1). + +## Modular GC API + +> [!WARNING] +> The Modular GC API is experimental and subject to change without notice. + +GC implementations interact with Ruby via the Modular GC API. All implementations must provide the functions in [gc/gc_impl.h](https://github.com/ruby/ruby/blob/master/gc/gc_impl.h) for Ruby to hook into. GC implementations can use any public C API in Ruby, along with additional APIs defined in [gc/gc.h](https://github.com/ruby/ruby/blob/master/gc/gc.h). + +Additionally, create an extconf.rb file to build the GC library. This file must use [gc/extconf_base.rb](https://github.com/ruby/ruby/blob/master/gc/extconf_base.rb) and the `create_gc_makefile` method. diff --git a/gc/default.c b/gc/default/default.c index f2915d7d8e..0027f7a13c 100644 --- a/gc/default.c +++ b/gc/default/default.c @@ -5,6 +5,9 @@ #ifndef _WIN32 # include <sys/mman.h> # include <unistd.h> +# ifdef HAVE_SYS_PRCTL_H +# include <sys/prctl.h> +# endif #endif #if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H) @@ -12,10 +15,15 @@ # include <sys/user.h> #endif -#include "internal/hash.h" +#ifdef BUILDING_MODULAR_GC +# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x)) +#else +# include "internal/bits.h" +#endif #include "ruby/ruby.h" #include "ruby/atomic.h" +#include "ruby_atomic.h" #include "ruby/debug.h" #include "ruby/thread.h" #include "ruby/util.h" @@ -26,12 +34,30 @@ #include "gc/gc.h" #include "gc/gc_impl.h" -#ifndef BUILDING_SHARED_GC +#ifndef BUILDING_MODULAR_GC # include "probes.h" #endif -#include "debug_counter.h" -#include "internal/sanitizers.h" +#ifdef BUILDING_MODULAR_GC +# define RB_DEBUG_COUNTER_INC(_name) ((void)0) +# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond)) +#else +# include "debug_counter.h" +#endif + +#ifdef BUILDING_MODULAR_GC +# define rb_asan_poison_object(obj) ((void)(obj)) +# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p)) +# define asan_unpoisoning_object(obj) if ((obj) || true) +# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size)) +# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p)) +# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true) + +# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size)) +# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size)) +#else +# include "internal/sanitizers.h" +#endif /* MALLOC_HEADERS_BEGIN */ #ifndef HAVE_MALLOC_USABLE_SIZE @@ -71,21 +97,12 @@ # include <mach/mach_port.h> #endif -#ifndef VM_CHECK_MODE -# define VM_CHECK_MODE RUBY_DEBUG -#endif - -// From ractor_core.h -#ifndef RACTOR_CHECK_MODE -# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE) -#endif - #ifndef RUBY_DEBUG_LOG # define RUBY_DEBUG_LOG(...) #endif -#ifndef GC_HEAP_INIT_SLOTS -#define GC_HEAP_INIT_SLOTS 10000 +#ifndef GC_HEAP_INIT_BYTES +#define GC_HEAP_INIT_BYTES (2560 * 1024) #endif #ifndef GC_HEAP_FREE_SLOTS #define GC_HEAP_FREE_SLOTS 4096 @@ -93,8 +110,8 @@ #ifndef GC_HEAP_GROWTH_FACTOR #define GC_HEAP_GROWTH_FACTOR 1.8 #endif -#ifndef GC_HEAP_GROWTH_MAX_SLOTS -#define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */ +#ifndef GC_HEAP_GROWTH_MAX_BYTES +#define GC_HEAP_GROWTH_MAX_BYTES 0 /* 0 is disable */ #endif #ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO # define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01 @@ -133,6 +150,17 @@ #define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */) #endif +#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD +#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 /* 8KB */) +#endif + +#ifdef RB_THREAD_LOCAL_SPECIFIER +#define USE_MALLOC_INCREASE_LOCAL 1 +static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local; +#else +#define USE_MALLOC_INCREASE_LOCAL 0 +#endif + #ifndef GC_CAN_COMPILE_COMPACTION #if defined(__wasi__) /* WebAssembly doesn't support signals */ # define GC_CAN_COMPILE_COMPACTION 0 @@ -150,25 +178,41 @@ #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) -#ifndef SIZE_POOL_COUNT -# define SIZE_POOL_COUNT 5 +#ifndef HEAP_COUNT +# if SIZEOF_VALUE >= 8 +# define HEAP_COUNT 12 +# else +# define HEAP_COUNT 5 +# endif +#endif + +/* The reciprocal table and pool_slot_sizes array are both generated from this + * single definition, so they can never get out of sync. */ +#if SIZEOF_VALUE >= 8 +# define EACH_POOL_SLOT_SIZE(SLOT) \ + SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \ + SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024) +#else +# define EACH_POOL_SLOT_SIZE(SLOT) \ + SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512) #endif -typedef struct ractor_newobj_size_pool_cache { +typedef struct ractor_newobj_heap_cache { struct free_slot *freelist; struct heap_page *using_page; -} rb_ractor_newobj_size_pool_cache_t; + size_t allocated_objects_count; +} rb_ractor_newobj_heap_cache_t; typedef struct ractor_newobj_cache { size_t incremental_mark_step_allocated_slots; - rb_ractor_newobj_size_pool_cache_t size_pool_caches[SIZE_POOL_COUNT]; + rb_ractor_newobj_heap_cache_t heap_caches[HEAP_COUNT]; } rb_ractor_newobj_cache_t; typedef struct { - size_t size_pool_init_slots[SIZE_POOL_COUNT]; + size_t heap_init_bytes; size_t heap_free_slots; double growth_factor; - size_t growth_max_slots; + size_t growth_max_bytes; double heap_free_slots_min_ratio; double heap_free_slots_goal_ratio; @@ -186,10 +230,10 @@ typedef struct { } ruby_gc_params_t; static ruby_gc_params_t gc_params = { - { GC_HEAP_INIT_SLOTS }, + GC_HEAP_INIT_BYTES, GC_HEAP_FREE_SLOTS, GC_HEAP_GROWTH_FACTOR, - GC_HEAP_GROWTH_MAX_SLOTS, + GC_HEAP_GROWTH_MAX_BYTES, GC_HEAP_FREE_SLOTS_MIN_RATIO, GC_HEAP_FREE_SLOTS_GOAL_RATIO, @@ -236,18 +280,6 @@ static ruby_gc_params_t gc_params = { #endif int ruby_rgengc_debug; -/* RGENGC_CHECK_MODE - * 0: disable all assertions - * 1: enable assertions (to debug RGenGC) - * 2: enable internal consistency check at each GC (for debugging) - * 3: enable internal consistency check at each GC steps (for debugging) - * 4: enable liveness check - * 5: show all references - */ -#ifndef RGENGC_CHECK_MODE -# define RGENGC_CHECK_MODE 0 -#endif - /* RGENGC_PROFILE * 0: disable RGenGC profiling * 1: enable profiling for basic information @@ -267,13 +299,6 @@ int ruby_rgengc_debug; # define RGENGC_ESTIMATE_OLDMALLOC 1 #endif -/* RGENGC_FORCE_MAJOR_GC - * Force major/full GC if this macro is not 0. - */ -#ifndef RGENGC_FORCE_MAJOR_GC -# define RGENGC_FORCE_MAJOR_GC 0 -#endif - #ifndef GC_PROFILE_MORE_DETAIL # define GC_PROFILE_MORE_DETAIL 0 #endif @@ -283,9 +308,24 @@ int ruby_rgengc_debug; #ifndef GC_ENABLE_LAZY_SWEEP # define GC_ENABLE_LAZY_SWEEP 1 #endif + +#ifndef VERIFY_FREE_SIZE +#if RUBY_DEBUG +#define VERIFY_FREE_SIZE 1 +#else +#define VERIFY_FREE_SIZE 0 +#endif +#endif + +#if VERIFY_FREE_SIZE +#undef CALC_EXACT_MALLOC_SIZE +#define CALC_EXACT_MALLOC_SIZE 1 +#endif + #ifndef CALC_EXACT_MALLOC_SIZE # define CALC_EXACT_MALLOC_SIZE 0 #endif + #if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0 # ifndef MALLOC_ALLOCATED_SIZE # define MALLOC_ALLOCATED_SIZE 0 @@ -378,7 +418,6 @@ struct RMoved { VALUE flags; VALUE dummy; VALUE destination; - uint32_t original_shape_id; }; #define RMOVED(obj) ((struct RMoved *)(obj)) @@ -415,22 +454,9 @@ typedef struct mark_stack { size_t unused_cache_size; } mark_stack_t; -#define SIZE_POOL_EDEN_HEAP(size_pool) (&(size_pool)->eden_heap) - typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d); typedef struct rb_heap_struct { - struct heap_page *free_pages; - struct ccan_list_head pages; - struct heap_page *sweeping_page; /* iterator for .pages */ - struct heap_page *compact_cursor; - uintptr_t compact_cursor_index; - struct heap_page *pooled_pages; - size_t total_pages; /* total page count in a heap */ - size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */ -} rb_heap_t; - -typedef struct rb_size_pool_struct { short slot_size; /* Basic statistics */ @@ -445,8 +471,16 @@ typedef struct rb_size_pool_struct { size_t freed_slots; size_t empty_slots; - rb_heap_t eden_heap; -} rb_size_pool_t; + struct heap_page *free_pages; + struct ccan_list_head pages; + struct heap_page *sweeping_page; /* iterator for .pages */ + struct heap_page *compact_cursor; + uintptr_t compact_cursor_index; + struct heap_page *pooled_pages; + size_t total_pages; /* total page count in a heap */ + size_t total_slots; /* total slot count */ + +} rb_heap_t; enum { gc_stress_no_major, @@ -462,10 +496,35 @@ enum gc_mode { gc_mode_compacting, }; +typedef rbimpl_atomic_uint64_t gc_counter_t; + +#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \ + !(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))) +# define MALLOC_COUNTERS_NEED_LOCK 1 +#endif + +struct gc_malloc_bytes { + gc_counter_t malloc; + gc_counter_t free; + + /* Snapshots of `malloc` / `free` taken at the end of the last GC */ + gc_counter_t malloc_at_last_gc; + gc_counter_t free_at_last_gc; +}; + typedef struct rb_objspace { struct { + struct gc_malloc_bytes counters; +#if RGENGC_ESTIMATE_OLDMALLOC + struct gc_malloc_bytes oldcounters; +#endif +#ifdef MALLOC_COUNTERS_NEED_LOCK + rb_nativethread_lock_t lock; +#endif + } malloc_counters; + + struct { size_t limit; - size_t increase; #if MALLOC_ALLOCATED_SIZE size_t allocated_size; size_t allocations; @@ -485,16 +544,14 @@ typedef struct rb_objspace { unsigned int during_compacting : 1; unsigned int during_reference_updating : 1; unsigned int gc_stressful: 1; - unsigned int has_newobj_hook: 1; unsigned int during_minor_gc : 1; unsigned int during_incremental_marking : 1; unsigned int measure_gc : 1; } flags; rb_event_flag_t hook_events; - unsigned long long next_object_id; - rb_size_pool_t size_pools[SIZE_POOL_COUNT]; + rb_heap_t heaps[HEAP_COUNT]; size_t empty_pages_count; struct heap_page *empty_pages; @@ -513,7 +570,7 @@ typedef struct rb_objspace { uintptr_t range[2]; size_t freeable_pages; - size_t allocatable_slots; + size_t allocatable_bytes; /* final */ VALUE deferred_final; @@ -560,23 +617,25 @@ typedef struct rb_objspace { double gc_sweep_start_time; size_t total_allocated_objects_at_gc_start; size_t heap_used_at_gc_start; + size_t heap_total_slots_at_gc_start; /* basic statistics */ size_t count; - uint64_t marking_time_ns; + unsigned long long marking_time_ns; struct timespec marking_start_time; - uint64_t sweeping_time_ns; + unsigned long long sweeping_time_ns; struct timespec sweeping_start_time; /* Weak references */ size_t weak_references_count; - size_t retained_weak_references_count; } profile; VALUE gc_stress_mode; struct { + bool parent_object_old_p; VALUE parent_object; + int need_major_gc; size_t last_major_gc; size_t uncollectible_wb_unprotected_objects; @@ -585,7 +644,6 @@ typedef struct rb_objspace { size_t old_objects_limit; #if RGENGC_ESTIMATE_OLDMALLOC - size_t oldmalloc_increase; size_t oldmalloc_increase_limit; #endif @@ -611,17 +669,20 @@ typedef struct rb_objspace { size_t step_slots; } rincgc; - st_table *id_to_obj_tbl; - st_table *obj_to_id_tbl; - #if GC_DEBUG_STRESS_TO_CLASS VALUE stress_to_class; #endif - rb_darray(VALUE *) weak_references; + rb_darray(VALUE) weak_references; rb_postponed_job_handle_t finalize_deferred_pjob; unsigned long live_ractor_cache_count; + + int sweeping_heap_count; + + int fork_vm_lock_lev; + + struct rb_gc_vm_context vm_context; } rb_objspace_t; #ifndef HEAP_PAGE_ALIGN_LOG @@ -629,10 +690,10 @@ typedef struct rb_objspace { #define HEAP_PAGE_ALIGN_LOG 16 #endif -#if RACTOR_CHECK_MODE || GC_DEBUG +#if RB_GC_OBJ_HAS_SUFFIX || GC_DEBUG struct rvalue_overhead { -# if RACTOR_CHECK_MODE - uint32_t _ractor_belonging_id; +# if RB_GC_OBJ_HAS_SUFFIX + struct rb_gc_obj_suffix suffix; # endif # if GC_DEBUG const char *file; @@ -650,10 +711,37 @@ struct rvalue_overhead { size_t rb_gc_impl_obj_slot_size(VALUE obj); # define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj))) #else -# define RVALUE_OVERHEAD 0 +# ifndef RVALUE_OVERHEAD +# define RVALUE_OVERHEAD 0 +# endif #endif -#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD) +#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD) + +static const size_t pool_slot_sizes[HEAP_COUNT] = { +#define SLOT(size) ((size) + RVALUE_OVERHEAD), + EACH_POOL_SLOT_SIZE(SLOT) +#undef SLOT +}; + +/* Precomputed reciprocals for fast slot index calculation. + * For slot size d: reciprocal = ceil(2^48 / d). + * Then offset / d == (uint32_t)((offset * reciprocal) >> 48) + * for all offset < HEAP_PAGE_SIZE. */ +#define SLOT_RECIPROCAL_SHIFT 48 +#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size)) + +static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = { +#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD), + EACH_POOL_SLOT_SIZE(SLOT) +#undef SLOT +}; + +#if SIZEOF_VALUE >= 8 +static uint8_t size_to_heap_idx[1024 / 8 + 1]; +#else +static uint8_t size_to_heap_idx[512 / 8 + 1]; +#endif #ifndef MAX # define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -663,12 +751,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj); #endif #define roomof(x, y) (((x) + (y) - 1) / (y)) #define CEILDIV(i, mod) roomof(i, mod) +#define MIN_POOL_SLOT_SIZE 32 enum { HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG), HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)), HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN, - HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)) / BASE_SLOT_SIZE), - HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH), + HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH), HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT), }; #define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG) @@ -730,6 +818,10 @@ struct free_slot { }; struct heap_page { + /* Cache line 0: allocation fast path + SLOT_INDEX */ + struct free_slot *freelist; + uintptr_t start; + uint64_t slot_size_reciprocal; unsigned short slot_size; unsigned short total_slots; unsigned short free_slots; @@ -741,12 +833,10 @@ struct heap_page { unsigned int has_uncollectible_wb_unprotected_objects : 1; } flags; - rb_size_pool_t *size_pool; + rb_heap_t *heap; struct heap_page *free_next; struct heap_page_body *body; - uintptr_t start; - struct free_slot *freelist; struct ccan_list_node page_node; bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT]; @@ -786,16 +876,18 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * if (page->total_slots == 0) { GC_ASSERT(page->start == 0); GC_ASSERT(page->slot_size == 0); - GC_ASSERT(page->size_pool == NULL); + GC_ASSERT(page->heap == NULL); GC_ASSERT(page->free_slots == 0); - GC_ASSERT(page->freelist == NULL); + asan_unpoisoning_memory_region(&page->freelist, sizeof(&page->freelist)) { + GC_ASSERT(page->freelist == NULL); + } return true; } else { GC_ASSERT(page->start != 0); GC_ASSERT(page->slot_size != 0); - GC_ASSERT(page->size_pool != NULL); + GC_ASSERT(page->heap != NULL); return false; } @@ -805,44 +897,63 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header) #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page) -#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE) -#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH ) -#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1)) -#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p)) +static inline size_t +slot_index_for_offset(size_t offset, uint64_t reciprocal) +{ + return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT); +} -/* Bitmap Operations */ -#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p)) -#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p)) -#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p)) +#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal) +#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH) +#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1)) +#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p)) + +#define _MARKED_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p)) +#define _MARK_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p)) +#define _CLEAR_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p)) + +#define MARKED_IN_BITMAP(bits, p) _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p) +#define MARK_IN_BITMAP(bits, p) _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p) +#define CLEAR_IN_BITMAP(bits, p) _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p) -/* getting bitmap */ #define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0]) #define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0]) #define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0]) #define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0]) #define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0]) -#define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3 - -#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) -#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT) - static int RVALUE_AGE_GET(VALUE obj) { - bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; - return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK; + struct heap_page *page = GET_HEAP_PAGE(obj); + bits_t *age_bits = page->age_bits; + size_t slot_idx = SLOT_INDEX(page, obj); + size_t idx = (slot_idx / BITS_BITLENGTH) * 2; + int shift = (int)(slot_idx & (BITS_BITLENGTH - 1)); + int lo = (age_bits[idx] >> shift) & 1; + int hi = (age_bits[idx + 1] >> shift) & 1; + return lo | (hi << 1); } static void -RVALUE_AGE_SET(VALUE obj, int age) +RVALUE_AGE_SET_BITMAP(VALUE obj, int age) { RUBY_ASSERT(age <= RVALUE_OLD_AGE); - bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; - // clear the bits - age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj))); - // shift the correct value in - age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj)); + struct heap_page *page = GET_HEAP_PAGE(obj); + bits_t *age_bits = page->age_bits; + size_t slot_idx = SLOT_INDEX(page, obj); + size_t idx = (slot_idx / BITS_BITLENGTH) * 2; + int shift = (int)(slot_idx & (BITS_BITLENGTH - 1)); + bits_t mask = (bits_t)1 << shift; + + age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift); + age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift); +} + +static void +RVALUE_AGE_SET(VALUE obj, int age) +{ + RVALUE_AGE_SET_BITMAP(obj, age); if (age == RVALUE_OLD_AGE) { RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED); } @@ -852,13 +963,117 @@ RVALUE_AGE_SET(VALUE obj, int age) } #define malloc_limit objspace->malloc_params.limit -#define malloc_increase objspace->malloc_params.increase +#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters) #define malloc_allocated_size objspace->malloc_params.allocated_size + +#ifdef MALLOC_COUNTERS_NEED_LOCK +# define MALLOC_COUNTERS_LOCK(o) rb_native_mutex_lock(&(o)->malloc_counters.lock) +# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock) +#else +# define MALLOC_COUNTERS_LOCK(o) ((void)0) +# define MALLOC_COUNTERS_UNLOCK(o) ((void)0) +#endif + +static inline void +gc_counter_add(gc_counter_t *p, size_t delta) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + *p += (gc_counter_t)delta; +#else + rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta); +#endif +} + +static inline gc_counter_t +gc_counter_load_relaxed(const gc_counter_t *p) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + return *p; +#else + return rbimpl_atomic_u64_load_relaxed(p); +#endif +} + +static inline gc_counter_t +gc_counter_load_acquire(const gc_counter_t *p) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + return *p; +#else + return rbimpl_atomic_u64_load_acquire(p); +#endif +} + +static inline void +gc_counter_store_release(gc_counter_t *p, gc_counter_t v) +{ +#ifdef MALLOC_COUNTERS_NEED_LOCK + *p = v; +#else + rbimpl_atomic_u64_set_release(p, v); +#endif +} + +static inline int64_t +gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c) +{ + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc); + gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc); + gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc); + gc_counter_t free_now = gc_counter_load_relaxed(&c->free); + MALLOC_COUNTERS_UNLOCK(objspace); + + gc_counter_t malloc_delta = malloc_now - malloc_at; + gc_counter_t free_delta = free_now - free_at; + + if (malloc_delta >= free_delta) { + return (int64_t)(malloc_delta - free_delta); + } + else { + return -(int64_t)(free_delta - malloc_delta); + } +} + +static inline size_t +gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c) +{ + int64_t inc = gc_malloc_counters_increase(objspace, c); + if (inc <= 0) return 0; +#if SIZEOF_SIZE_T < 8 + if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX; +#endif + return (size_t)inc; +} + +static inline int64_t +gc_malloc_counters_snapshot(rb_objspace_t *objspace, struct gc_malloc_bytes *c) +{ + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc); + gc_counter_t free_now = gc_counter_load_relaxed(&c->free); + gc_counter_t malloc_at = gc_counter_load_relaxed(&c->malloc_at_last_gc); + gc_counter_t free_at = gc_counter_load_relaxed(&c->free_at_last_gc); + gc_counter_store_release(&c->malloc_at_last_gc, malloc_now); + gc_counter_store_release(&c->free_at_last_gc, free_now); + MALLOC_COUNTERS_UNLOCK(objspace); + + gc_counter_t malloc_delta = malloc_now - malloc_at; + gc_counter_t free_delta = free_now - free_at; + + if (malloc_delta >= free_delta) { + return (int64_t)(malloc_delta - free_delta); + } + else { + return -(int64_t)(free_delta - malloc_delta); + } +} + #define heap_pages_lomem objspace->heap_pages.range[0] #define heap_pages_himem objspace->heap_pages.range[1] #define heap_pages_freeable_pages objspace->heap_pages.freeable_pages #define heap_pages_deferred_final objspace->heap_pages.deferred_final -#define size_pools objspace->size_pools +#define heaps objspace->heaps #define during_gc objspace->flags.during_gc #define finalizing objspace->atomic_flags.finalizing #define finalizer_table objspace->finalizer_table @@ -868,8 +1083,8 @@ RVALUE_AGE_SET(VALUE obj, int age) #define stress_to_class objspace->stress_to_class #define set_stress_to_class(c) (stress_to_class = (c)) #else -#define stress_to_class (objspace, 0) -#define set_stress_to_class(c) (objspace, (c)) +#define stress_to_class ((void)objspace, 0) +#define set_stress_to_class(c) ((void)objspace, (c)) #endif #if 0 @@ -919,30 +1134,15 @@ gc_mode_verify(enum gc_mode mode) static inline bool has_sweeping_pages(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - if (SIZE_POOL_EDEN_HEAP(&size_pools[i])->sweeping_page) { - return TRUE; - } - } - return FALSE; + return objspace->sweeping_heap_count != 0; } static inline size_t heap_eden_total_pages(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_pages; - } - return count; -} - -static inline size_t -heap_eden_total_slots(rb_objspace_t *objspace) -{ - size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_slots; + for (int i = 0; i < HEAP_COUNT; i++) { + count += (&heaps[i])->total_pages; } return count; } @@ -951,9 +1151,9 @@ static inline size_t total_allocated_objects(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - count += size_pool->total_allocated_objects; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + count += heap->total_allocated_objects; } return count; } @@ -962,9 +1162,9 @@ static inline size_t total_freed_objects(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - count += size_pool->total_freed_objects; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + count += heap->total_freed_objects; } return count; } @@ -973,9 +1173,9 @@ static inline size_t total_final_slots_count(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - count += size_pool->final_slots_count; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + count += heap->final_slots_count; } return count; } @@ -989,9 +1189,22 @@ total_final_slots_count(rb_objspace_t *objspace) #define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE) #define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE) #define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE) -#define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048 -#define GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT 1024 +/* + * Byte budget for incremental sweep steps. Each step sweeps at most + * this many bytes worth of slots before yielding. The effective slot + * count per step is GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size, + * so larger slot pools (which are less heavily used) naturally get + * fewer slots swept per step. + * + * Baseline: 2048 slots * RVALUE_SLOT_SIZE = 2048 * 40 = 81920 bytes, + * preserving the historical behavior for the smallest heap. + */ +#define GC_INCREMENTAL_SWEEP_BYTES (2048 * RVALUE_SLOT_SIZE) +#define GC_INCREMENTAL_SWEEP_POOL_BYTES (1024 * RVALUE_SLOT_SIZE) #define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace)) +/* In lazy sweeping or the previous incremental marking finished and did not yield a free page. */ +#define needs_continue_sweeping(objspace, heap) \ + ((heap)->free_pages == NULL && is_lazy_sweeping(objspace)) #if SIZEOF_LONG == SIZEOF_VOIDP # define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */ @@ -1003,7 +1216,7 @@ total_final_slots_count(rb_objspace_t *objspace) #endif struct RZombie { - struct RBasic basic; + VALUE flags; VALUE next; void (*dfree)(void *); void *data; @@ -1011,10 +1224,9 @@ struct RZombie { #define RZOMBIE(o) ((struct RZombie *)(o)) -int ruby_disable_gc = 0; -int ruby_enable_autocompact = 0; +static bool ruby_enable_autocompact = false; #if RGENGC_CHECK_MODE -gc_compact_compare_func ruby_autocompact_compare_func; +static gc_compact_compare_func ruby_autocompact_compare_func; #endif static void init_mark_stack(mark_stack_t *stack); @@ -1036,11 +1248,11 @@ static void gc_marking_enter(rb_objspace_t *objspace); static void gc_marking_exit(rb_objspace_t *objspace); static void gc_sweeping_enter(rb_objspace_t *objspace); static void gc_sweeping_exit(rb_objspace_t *objspace); -static bool gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap); +static bool gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap); static void gc_sweep(rb_objspace_t *objspace); -static void gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool); -static void gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap); +static void gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap); +static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap); static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr); static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr); @@ -1118,10 +1330,10 @@ tick(void) return val; } +#elif defined(__POWERPC__) && defined(__APPLE__) /* Implementation for macOS PPC by @nobu * See: https://github.com/ruby/ruby/pull/5975#discussion_r890045558 */ -#elif defined(__POWERPC__) && defined(__APPLE__) typedef unsigned long long tick_t; #define PRItick "llu" @@ -1177,19 +1389,6 @@ tick(void) #define MEASURE_LINE(expr) expr #endif /* USE_TICK_T */ -#define asan_unpoisoning_object(obj) \ - for (void *poisoned = asan_unpoison_object_temporary(obj), \ - *unpoisoning = &poisoned; /* flag to loop just once */ \ - unpoisoning; \ - unpoisoning = asan_poison_object_restore(obj, poisoned)) - -#define FL_CHECK2(name, x, pred) \ - ((RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) ? \ - (rb_bug(name": SPECIAL_CONST (%p)", (void *)(x)), 0) : (pred)) -#define FL_TEST2(x,f) FL_CHECK2("FL_TEST2", x, FL_TEST_RAW((x),(f)) != 0) -#define FL_SET2(x,f) FL_CHECK2("FL_SET2", x, RBASIC(x)->flags |= (f)) -#define FL_UNSET2(x,f) FL_CHECK2("FL_UNSET2", x, RBASIC(x)->flags &= ~(f)) - static inline VALUE check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj); #define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj)) @@ -1253,7 +1452,7 @@ check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int ter { int err = 0; - int lev = rb_gc_vm_lock_no_barrier(); + int lev = RB_GC_VM_LOCK_NO_BARRIER(); { if (SPECIAL_CONST_P(obj)) { fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj); @@ -1343,7 +1542,7 @@ check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int ter } } } - rb_gc_vm_unlock_no_barrier(lev); + RB_GC_VM_UNLOCK_NO_BARRIER(lev); if (err > 0 && terminate) { rb_bug("check_rvalue_consistency_force: there is %d errors.", err); @@ -1369,20 +1568,12 @@ check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj) static inline bool gc_object_moved_p(rb_objspace_t *objspace, VALUE obj) { - if (RB_SPECIAL_CONST_P(obj)) { - return FALSE; - } - else { - void *poisoned = asan_unpoison_object_temporary(obj); - int ret = BUILTIN_TYPE(obj) == T_MOVED; - /* Re-poison slot if it's not the one we want */ - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE); - asan_poison_object(obj); - } - return ret; + bool ret; + asan_unpoisoning_object(obj) { + ret = BUILTIN_TYPE(obj) == T_MOVED; } + return ret; } static inline int @@ -1523,147 +1714,66 @@ rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) { rb_objspace_t *objspace = objspace_ptr; objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK; - objspace->flags.has_newobj_hook = !!(objspace->hook_events & RUBY_INTERNAL_EVENT_NEWOBJ); } -VALUE -rb_gc_impl_get_profile_total_time(void *objspace_ptr) +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - uint64_t marking_time = objspace->profile.marking_time_ns; - uint64_t sweeping_time = objspace->profile.sweeping_time_ns; + unsigned long long marking_time = objspace->profile.marking_time_ns; + unsigned long long sweeping_time = objspace->profile.sweeping_time_ns; - return ULL2NUM(marking_time + sweeping_time); + return marking_time + sweeping_time; } -VALUE +void rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) { rb_objspace_t *objspace = objspace_ptr; objspace->flags.measure_gc = RTEST(flag) ? TRUE : FALSE; - - return flag; } -VALUE +bool rb_gc_impl_get_measure_total_time(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - return objspace->flags.measure_gc ? Qtrue : Qfalse; -} - -static size_t -minimum_slots_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) -{ - size_t size_pool_idx = size_pool - size_pools; - return gc_params.size_pool_init_slots[size_pool_idx]; -} - -static VALUE initial_stress = Qfalse; - -void -rb_gc_impl_initial_stress_set(VALUE flag) -{ - initial_stress = flag; -} - -static int -object_id_cmp(st_data_t x, st_data_t y) -{ - if (RB_TYPE_P(x, T_BIGNUM)) { - return !rb_big_eql(x, y); - } - else { - return x != y; - } -} - -static st_index_t -object_id_hash(st_data_t n) -{ - return FIX2LONG(rb_hash((VALUE)n)); + return objspace->flags.measure_gc; } -#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1) -#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT) - -static const struct st_hash_type object_id_hash_type = { - object_id_cmp, - object_id_hash, -}; - /* garbage objects will be collected soon. */ bool rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr) { rb_objspace_t *objspace = objspace_ptr; + bool dead = false; + asan_unpoisoning_object(ptr) { switch (BUILTIN_TYPE(ptr)) { case T_NONE: case T_MOVED: case T_ZOMBIE: - return true; + dead = true; + break; default: break; } } + if (dead) return true; return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep && !RVALUE_MARKED(objspace, ptr); } -VALUE -rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id) +struct rb_gc_vm_context * +rb_gc_impl_get_vm_context(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - VALUE obj; - if (st_lookup(objspace->id_to_obj_tbl, object_id, &obj) && - !rb_gc_impl_garbage_object_p(objspace, obj)) { - return obj; - } - - if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(objspace->next_object_id))) { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); - } - else { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); - } -} - -VALUE -rb_gc_impl_object_id(void *objspace_ptr, VALUE obj) -{ - VALUE id; - rb_objspace_t *objspace = objspace_ptr; - - unsigned int lev = rb_gc_vm_lock(); - if (FL_TEST(obj, FL_SEEN_OBJ_ID)) { - st_data_t val; - if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &val)) { - id = (VALUE)val; - } - else { - rb_bug("rb_gc_impl_object_id: FL_SEEN_OBJ_ID flag set but not found in table"); - } - } - else { - GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, NULL)); - - id = ULL2NUM(objspace->next_object_id); - objspace->next_object_id += OBJ_ID_INCREMENT; - - st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id); - st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj); - FL_SET(obj, FL_SEEN_OBJ_ID); - } - rb_gc_vm_unlock(lev); - - return id; + return &objspace->vm_context; } static void free_stack_chunks(mark_stack_t *); @@ -1673,7 +1783,7 @@ static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page); static inline void heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) { - asan_unpoison_object(obj, false); + rb_asan_unpoison_object(obj, false); asan_unlock_freelist(page); @@ -1683,26 +1793,24 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj page->freelist = slot; asan_lock_freelist(page); - RVALUE_AGE_RESET(obj); + // Should have already been reset + GC_ASSERT(RVALUE_AGE_GET(obj) == 0); if (RGENGC_CHECK_MODE && /* obj should belong to page */ !(page->start <= (uintptr_t)obj && (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) && - obj % BASE_SLOT_SIZE == 0)) { + obj % sizeof(VALUE) == 0)) { rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj); } - asan_poison_object(obj); + rb_asan_poison_object(obj); gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj); } -static size_t -objspace_available_slots(rb_objspace_t *objspace); - static void -size_pool_allocatable_slots_expand(rb_objspace_t *objspace, - rb_size_pool_t *size_pool, size_t free_slots, size_t total_slots) +heap_allocatable_bytes_expand(rb_objspace_t *objspace, + rb_heap_t *heap, size_t free_slots, size_t total_slots, size_t slot_size) { double goal_ratio = gc_params.heap_free_slots_goal_ratio; size_t target_total_slots; @@ -1711,7 +1819,7 @@ size_pool_allocatable_slots_expand(rb_objspace_t *objspace, target_total_slots = (size_t)(total_slots * gc_params.growth_factor); } else if (total_slots == 0) { - target_total_slots = minimum_slots_for_size_pool(objspace, size_pool); + target_total_slots = gc_params.heap_init_bytes / slot_size; } else { /* Find `f' where free_slots = f * total_slots * goal_ratio @@ -1734,8 +1842,8 @@ size_pool_allocatable_slots_expand(rb_objspace_t *objspace, } } - if (gc_params.growth_max_slots > 0) { - size_t max_total_slots = (size_t)(total_slots + gc_params.growth_max_slots); + if (gc_params.growth_max_bytes > 0) { + size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size; if (target_total_slots > max_total_slots) target_total_slots = max_total_slots; } @@ -1743,7 +1851,7 @@ size_pool_allocatable_slots_expand(rb_objspace_t *objspace, /* Extend by at least 1 page. */ if (extend_slot_count == 0) extend_slot_count = 1; - objspace->heap_pages.allocatable_slots += extend_slot_count; + objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size; } static inline void @@ -1826,13 +1934,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page) static void heap_pages_free_unused_pages(rb_objspace_t *objspace) { - size_t pages_to_keep_count = - // Get number of pages estimated for the smallest size pool - CEILDIV(objspace->heap_pages.allocatable_slots, HEAP_PAGE_OBJ_LIMIT) * - // Estimate the average slot size multiple - (1 << (SIZE_POOL_COUNT / 2)); - - if (objspace->empty_pages != NULL && objspace->empty_pages_count > pages_to_keep_count) { + if (objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) { GC_ASSERT(objspace->empty_pages_count > 0); objspace->empty_pages = NULL; objspace->empty_pages_count = 0; @@ -1841,15 +1943,15 @@ heap_pages_free_unused_pages(rb_objspace_t *objspace) for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); - if (heap_page_in_global_empty_pages_pool(objspace, page) && pages_to_keep_count == 0) { + if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) { heap_page_free(objspace, page); + heap_pages_freeable_pages--; } else { - if (heap_page_in_global_empty_pages_pool(objspace, page) && pages_to_keep_count > 0) { + if (heap_page_in_global_empty_pages_pool(objspace, page)) { page->free_next = objspace->empty_pages; objspace->empty_pages = page; objspace->empty_pages_count++; - pages_to_keep_count--; } if (i != j) { @@ -1917,12 +2019,23 @@ heap_page_body_allocate(void) #ifdef HAVE_MMAP GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0); - char *ptr = mmap(NULL, HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE, + size_t mmap_size = HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE; + char *ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (ptr == MAP_FAILED) { return NULL; } + // If we are building `default.c` as part of the ruby executable, we + // may just call `ruby_annotate_mmap`. But if we are building + // `default.c` as a shared library, we will not have access to private + // symbols, and we have to either call prctl directly or make our own + // wrapper. +#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME) + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size, "Ruby:GC:default:heap_page_body_allocate"); + errno = 0; +#endif + char *aligned = ptr + HEAP_PAGE_ALIGN; aligned -= ((VALUE)aligned & (HEAP_PAGE_ALIGN - 1)); GC_ASSERT(aligned > ptr); @@ -1960,7 +2073,10 @@ static struct heap_page * heap_page_resurrect(rb_objspace_t *objspace) { struct heap_page *page = NULL; - if (objspace->empty_pages != NULL) { + if (objspace->empty_pages == NULL) { + GC_ASSERT(objspace->empty_pages_count == 0); + } + else { GC_ASSERT(objspace->empty_pages_count > 0); objspace->empty_pages_count--; page = objspace->empty_pages; @@ -2005,7 +2121,7 @@ heap_page_allocate(rb_objspace_t *objspace) } } - rb_darray_insert(&objspace->heap_pages.sorted, hi, page); + rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page); if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start; if (heap_pages_himem < end) heap_pages_himem = end; @@ -2019,47 +2135,39 @@ heap_page_allocate(rb_objspace_t *objspace) } static void -size_pool_add_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, struct heap_page *page) +heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) { /* Adding to eden heap during incremental sweeping is forbidden */ GC_ASSERT(!heap->sweeping_page); GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); - /* adjust obj_limit (object number available in this page) */ + /* Align start to slot_size boundary */ uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header); - if (start % BASE_SLOT_SIZE != 0) { - int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE); - start = start + delta; - GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1); - - /* Find a num in page that is evenly divisible by `stride`. - * This is to ensure that objects are aligned with bit planes. - * In other words, ensure there are an even number of objects - * per bit plane. */ - if (NUM_IN_PAGE(start) == 1) { - start += size_pool->slot_size - BASE_SLOT_SIZE; - } - - GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % size_pool->slot_size == 0); - } + uintptr_t rem = start % heap->slot_size; + if (rem) start += heap->slot_size - rem; - int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/size_pool->slot_size); + int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size); page->start = start; page->total_slots = slot_count; - page->slot_size = size_pool->slot_size; - page->size_pool = size_pool; + page->slot_size = heap->slot_size; + page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps]; + page->heap = heap; + + memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); + memset(&page->age_bits[0], 0, sizeof(page->age_bits)); asan_unlock_freelist(page); page->freelist = NULL; - for (VALUE p = (VALUE)start; p < start + (slot_count * size_pool->slot_size); p += size_pool->slot_size) { + asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false); + for (VALUE p = (VALUE)start; p < start + (slot_count * heap->slot_size); p += heap->slot_size) { heap_page_add_freeobj(objspace, page, p); } asan_lock_freelist(page); page->free_slots = slot_count; - size_pool->total_allocated_pages++; + heap->total_allocated_pages++; ccan_list_add_tail(&heap->pages, &page->page_node); heap->total_pages++; @@ -2067,105 +2175,115 @@ size_pool_add_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t } static int -heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap) { - if (objspace->heap_pages.allocatable_slots > 0) { - gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", " - "allocatable_slots: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n", - rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_slots, heap->total_pages); + gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", " + "allocatable_bytes: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n", + rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_bytes, heap->total_pages); - struct heap_page *page = heap_page_resurrect(objspace); - if (page == NULL) { - page = heap_page_allocate(objspace); - } - size_pool_add_page(objspace, size_pool, heap, page); + bool allocated = false; + struct heap_page *page = heap_page_resurrect(objspace); + + if (page == NULL && objspace->heap_pages.allocatable_bytes > 0) { + page = heap_page_allocate(objspace); + allocated = true; + + GC_ASSERT(page != NULL); + } + + if (page != NULL) { + heap_add_page(objspace, heap, page); heap_add_freepage(heap, page); - if (objspace->heap_pages.allocatable_slots > (size_t)page->total_slots) { - objspace->heap_pages.allocatable_slots -= page->total_slots; - } - else { - objspace->heap_pages.allocatable_slots = 0; + if (allocated) { + size_t page_bytes = (size_t)page->total_slots * page->slot_size; + if (objspace->heap_pages.allocatable_bytes > page_bytes) { + objspace->heap_pages.allocatable_bytes -= page_bytes; + } + else { + objspace->heap_pages.allocatable_bytes = 0; + } } - - return true; } - return false; + return page != NULL; } static void -heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap) { - size_t prev_allocatable_slots = objspace->heap_pages.allocatable_slots; - // Set allocatable slots to 1 to force a page to be created. - objspace->heap_pages.allocatable_slots = 1; - heap_page_allocate_and_initialize(objspace, size_pool, heap); + size_t prev_allocatable_bytes = objspace->heap_pages.allocatable_bytes; + objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE; + heap_page_allocate_and_initialize(objspace, heap); GC_ASSERT(heap->free_pages != NULL); - objspace->heap_pages.allocatable_slots = prev_allocatable_slots; + objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes; } static void -gc_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +gc_continue(rb_objspace_t *objspace, rb_heap_t *heap) { unsigned int lock_lev; - gc_enter(objspace, gc_enter_event_continue, &lock_lev); + bool needs_gc = is_incremental_marking(objspace) || needs_continue_sweeping(objspace, heap); + if (!needs_gc) return; + + gc_enter(objspace, gc_enter_event_continue, &lock_lev); // takes vm barrier, try to avoid /* Continue marking if in incremental marking. */ if (is_incremental_marking(objspace)) { - if (gc_marks_continue(objspace, size_pool, heap)) { + if (gc_marks_continue(objspace, heap)) { gc_sweep(objspace); } } - /* Continue sweeping if in lazy sweeping or the previous incremental - * marking finished and did not yield a free page. */ - if (heap->free_pages == NULL && is_lazy_sweeping(objspace)) { - gc_sweep_continue(objspace, size_pool, heap); + if (needs_continue_sweeping(objspace, heap)) { + gc_sweep_continue(objspace, heap); } gc_exit(objspace, gc_enter_event_continue, &lock_lev); } static void -heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap) { GC_ASSERT(heap->free_pages == NULL); - if (SIZE_POOL_EDEN_HEAP(size_pool)->total_slots < gc_params.size_pool_init_slots[size_pool - size_pools] && - size_pool->eden_heap.sweeping_page == NULL) { - heap_page_allocate_and_initialize_force(objspace, size_pool, heap); + if (heap->total_slots < gc_params.heap_init_bytes / heap->slot_size && + heap->sweeping_page == NULL) { + heap_page_allocate_and_initialize_force(objspace, heap); GC_ASSERT(heap->free_pages != NULL); return; } /* Continue incremental marking or lazy sweeping, if in any of those steps. */ - gc_continue(objspace, size_pool, heap); + gc_continue(objspace, heap); if (heap->free_pages == NULL) { - heap_page_allocate_and_initialize(objspace, size_pool, heap); + heap_page_allocate_and_initialize(objspace, heap); } /* If we still don't have a free page and not allowed to create a new page, * we should start a new GC cycle. */ if (heap->free_pages == NULL) { + GC_ASSERT(objspace->empty_pages_count == 0); + GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0); + if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) { rb_memerror(); } else { - if (objspace->heap_pages.allocatable_slots == 0 && !gc_config_full_mark_val) { - size_pool_allocatable_slots_expand(objspace, size_pool, - size_pool->freed_slots + size_pool->empty_slots, - heap->total_slots); - GC_ASSERT(objspace->heap_pages.allocatable_slots > 0); + if (objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) { + heap_allocatable_bytes_expand(objspace, heap, + heap->freed_slots + heap->empty_slots, + heap->total_slots, heap->slot_size); + GC_ASSERT(objspace->heap_pages.allocatable_bytes > 0); } /* Do steps of incremental marking or lazy sweeping if the GC run permits. */ - gc_continue(objspace, size_pool, heap); + gc_continue(objspace, heap); /* If we're not incremental marking (e.g. a minor GC) or finished * sweeping and still don't have a free page, then - * gc_sweep_finish_size_pool should allow us to create a new page. */ - if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, size_pool, heap)) { + * gc_sweep_finish_heap should allow us to create a new page. */ + if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, heap)) { if (gc_needs_major_flags == GPR_FLAG_NONE) { rb_bug("cannot create a new page after GC"); } @@ -2175,10 +2293,10 @@ heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap } else { /* Do steps of incremental marking or lazy sweeping. */ - gc_continue(objspace, size_pool, heap); + gc_continue(objspace, heap); if (heap->free_pages == NULL && - !heap_page_allocate_and_initialize(objspace, size_pool, heap)) { + !heap_page_allocate_and_initialize(objspace, heap)) { rb_bug("cannot create a new page after major GC"); } } @@ -2190,16 +2308,6 @@ heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap GC_ASSERT(heap->free_pages != NULL); } -static inline VALUE -newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3) -{ - VALUE *p = (VALUE *)obj; - p[2] = v1; - p[3] = v2; - p[4] = v3; - return obj; -} - #if GC_DEBUG static inline const char* rb_gc_impl_source_location_cstr(int *ptr) @@ -2221,27 +2329,16 @@ rb_gc_impl_source_location_cstr(int *ptr) static inline VALUE newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj) { -#if !__has_feature(memory_sanitizer) GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE); GC_ASSERT((flags & FL_WB_PROTECTED) == 0); -#endif RBASIC(obj)->flags = flags; *((VALUE *)&RBASIC(obj)->klass) = klass; - - int t = flags & RUBY_T_MASK; - if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) { - RVALUE_AGE_SET_CANDIDATE(objspace, obj); - } - -#if RACTOR_CHECK_MODE - void rb_ractor_setup_belonging(VALUE obj); - rb_ractor_setup_belonging(obj); +#if RBASIC_SHAPE_ID_FIELD + RBASIC(obj)->shape_id = 0; #endif #if RGENGC_CHECK_MODE - newobj_fill(obj, 0, 0, 0); - - int lev = rb_gc_vm_lock_no_barrier(); + int lev = RB_GC_VM_LOCK_NO_BARRIER(); { check_rvalue_consistency(objspace, obj); @@ -2252,7 +2349,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj)); } - rb_gc_vm_unlock_no_barrier(lev); + RB_GC_VM_UNLOCK_NO_BARRIER(lev); #endif if (RB_UNLIKELY(wb_protected == FALSE)) { @@ -2281,7 +2378,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj)); - RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj)); + // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj)); return obj; } @@ -2292,34 +2389,38 @@ rb_gc_impl_obj_slot_size(VALUE obj) } static inline size_t -size_pool_slot_size(unsigned char pool_id) +heap_slot_size(unsigned char pool_id) { - GC_ASSERT(pool_id < SIZE_POOL_COUNT); - - size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE; + GC_ASSERT(pool_id < HEAP_COUNT); -#if RGENGC_CHECK_MODE - rb_objspace_t *objspace = rb_gc_get_objspace(); - GC_ASSERT(size_pools[pool_id].slot_size == (short)slot_size); -#endif - - slot_size -= RVALUE_OVERHEAD; - - return slot_size; + return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD; } bool rb_gc_impl_size_allocatable_p(size_t size) { - return size <= size_pool_slot_size(SIZE_POOL_COUNT - 1); + return size + RVALUE_OVERHEAD <= pool_slot_sizes[HEAP_COUNT - 1]; +} + +static const size_t ALLOCATED_COUNT_STEP = 1024; +static void +ractor_cache_flush_count(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache) +{ + for (int heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) { + rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx]; + + rb_heap_t *heap = &heaps[heap_idx]; + RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count); + heap_cache->allocated_objects_count = 0; + } } static inline VALUE ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, - size_t size_pool_idx) + size_t heap_idx) { - rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx]; - struct free_slot *p = size_pool_cache->freelist; + rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx]; + struct free_slot *p = heap_cache->freelist; if (RB_UNLIKELY(is_incremental_marking(objspace))) { // Not allowed to allocate without running an incremental marking step @@ -2334,13 +2435,20 @@ ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *ca if (RB_LIKELY(p)) { VALUE obj = (VALUE)p; - MAYBE_UNUSED(const size_t) stride = size_pool_slot_size(size_pool_idx); - size_pool_cache->freelist = p->next; - asan_unpoison_memory_region(p, stride, true); + rb_asan_unpoison_object(obj, true); + heap_cache->freelist = p->next; + + heap_cache->allocated_objects_count++; + rb_heap_t *heap = &heaps[heap_idx]; + if (heap_cache->allocated_objects_count >= ALLOCATED_COUNT_STEP) { + RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count); + heap_cache->allocated_objects_count = 0; + } + #if RGENGC_CHECK_MODE - GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == stride); + GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx)); // zero clear - MEMZERO((char *)obj, char, stride); + MEMZERO((char *)obj, char, heap_slot_size(heap_idx)); #endif return obj; } @@ -2350,12 +2458,12 @@ ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *ca } static struct heap_page * -heap_next_free_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +heap_next_free_page(rb_objspace_t *objspace, rb_heap_t *heap) { struct heap_page *page; if (heap->free_pages == NULL) { - heap_prepare(objspace, size_pool, heap); + heap_prepare(objspace, heap); } page = heap->free_pages; @@ -2369,111 +2477,111 @@ heap_next_free_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_ } static inline void -ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, +ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, struct heap_page *page) { gc_report(3, objspace, "ractor_set_cache: Using page %p\n", (void *)page->body); - rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx]; + rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx]; - GC_ASSERT(size_pool_cache->freelist == NULL); + GC_ASSERT(heap_cache->freelist == NULL); GC_ASSERT(page->free_slots != 0); GC_ASSERT(page->freelist != NULL); - size_pool_cache->using_page = page; - size_pool_cache->freelist = page->freelist; + heap_cache->using_page = page; + heap_cache->freelist = page->freelist; page->free_slots = 0; page->freelist = NULL; - asan_unpoison_object((VALUE)size_pool_cache->freelist, false); - GC_ASSERT(RB_TYPE_P((VALUE)size_pool_cache->freelist, T_NONE)); - asan_poison_object((VALUE)size_pool_cache->freelist); + rb_asan_unpoison_object((VALUE)heap_cache->freelist, false); + GC_ASSERT(RB_TYPE_P((VALUE)heap_cache->freelist, T_NONE)); + rb_asan_poison_object((VALUE)heap_cache->freelist); } -static inline size_t -size_pool_idx_for_size(size_t size) +static void +init_size_to_heap_idx(void) { - size += RVALUE_OVERHEAD; - - size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE); - - /* size_pool_idx is ceil(log2(slot_count)) */ - size_t size_pool_idx = 64 - nlz_int64(slot_count - 1); - - if (size_pool_idx >= SIZE_POOL_COUNT) { - rb_bug("size_pool_idx_for_size: allocation size too large " - "(size=%"PRIuSIZE"u, size_pool_idx=%"PRIuSIZE"u)", size, size_pool_idx); + for (size_t i = 0; i < sizeof(size_to_heap_idx); i++) { + size_t effective = i * 8 + RVALUE_OVERHEAD; + uint8_t idx; + for (idx = 0; idx < HEAP_COUNT; idx++) { + if (effective <= pool_slot_sizes[idx]) break; + } + size_to_heap_idx[i] = idx; } +} -#if RGENGC_CHECK_MODE - rb_objspace_t *objspace = rb_gc_get_objspace(); - GC_ASSERT(size <= (size_t)size_pools[size_pool_idx].slot_size); - if (size_pool_idx > 0) GC_ASSERT(size > (size_t)size_pools[size_pool_idx - 1].slot_size); -#endif +static inline size_t +heap_idx_for_size(size_t size) +{ + size_t compressed = (size + 7) >> 3; + if (compressed < sizeof(size_to_heap_idx)) { + size_t heap_idx = size_to_heap_idx[compressed]; + if (RB_LIKELY(heap_idx < HEAP_COUNT)) return heap_idx; + } - return size_pool_idx; + rb_bug("heap_idx_for_size: allocation size too large " + "(size=%"PRIuSIZE")", size); } size_t -rb_gc_impl_size_pool_id_for_size(void *objspace_ptr, size_t size) +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) { - return size_pool_idx_for_size(size); + return heap_idx_for_size(size); } -static size_t size_pool_sizes[SIZE_POOL_COUNT + 1] = { 0 }; +static size_t heap_sizes[HEAP_COUNT + 1] = { 0 }; size_t * -rb_gc_impl_size_pool_sizes(void *objspace_ptr) +rb_gc_impl_heap_sizes(void *objspace_ptr) { - if (size_pool_sizes[0] == 0) { - for (unsigned char i = 0; i < SIZE_POOL_COUNT; i++) { - size_pool_sizes[i] = size_pool_slot_size(i); + if (heap_sizes[0] == 0) { + for (unsigned char i = 0; i < HEAP_COUNT; i++) { + heap_sizes[i] = heap_slot_size(i); } } - return size_pool_sizes; + return heap_sizes; } -NOINLINE(static VALUE newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked)); +NOINLINE(static VALUE newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked)); static VALUE -newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked) +newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked) { - rb_size_pool_t *size_pool = &size_pools[size_pool_idx]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + rb_heap_t *heap = &heaps[heap_idx]; VALUE obj = Qfalse; unsigned int lev = 0; bool unlock_vm = false; if (!vm_locked) { - lev = rb_gc_cr_lock(); - vm_locked = true; + lev = RB_GC_CR_LOCK(); unlock_vm = true; } { if (is_incremental_marking(objspace)) { - gc_continue(objspace, size_pool, heap); + gc_continue(objspace, heap); cache->incremental_mark_step_allocated_slots = 0; // Retry allocation after resetting incremental_mark_step_allocated_slots - obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); + obj = ractor_cache_allocate_slot(objspace, cache, heap_idx); } if (obj == Qfalse) { // Get next free page (possibly running GC) - struct heap_page *page = heap_next_free_page(objspace, size_pool, heap); - ractor_cache_set_page(objspace, cache, size_pool_idx, page); + struct heap_page *page = heap_next_free_page(objspace, heap); + ractor_cache_set_page(objspace, cache, heap_idx, page); // Retry allocation after moving to new page - obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); + obj = ractor_cache_allocate_slot(objspace, cache, heap_idx); } } if (unlock_vm) { - rb_gc_cr_unlock(lev); + RB_GC_CR_UNLOCK(lev); } if (RB_UNLIKELY(obj == Qfalse)) { @@ -2483,37 +2591,34 @@ newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size } static VALUE -newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked) +newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked) { - VALUE obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); + VALUE obj = ractor_cache_allocate_slot(objspace, cache, heap_idx); if (RB_UNLIKELY(obj == Qfalse)) { - obj = newobj_cache_miss(objspace, cache, size_pool_idx, vm_locked); + obj = newobj_cache_miss(objspace, cache, heap_idx, vm_locked); } - rb_size_pool_t *size_pool = &size_pools[size_pool_idx]; - size_pool->total_allocated_objects++; - GC_ASSERT(rb_gc_multi_ractor_p() || - SIZE_POOL_EDEN_HEAP(size_pool)->total_slots >= - (size_pool->total_allocated_objects - size_pool->total_freed_objects - size_pool->final_slots_count)); - return obj; } -ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t size_pool_idx)); +ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t heap_idx)); static inline VALUE -newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t size_pool_idx) +newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t heap_idx) { VALUE obj; unsigned int lev; - lev = rb_gc_cr_lock(); + lev = RB_GC_CR_LOCK(); { if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) { if (during_gc) { dont_gc_on(); during_gc = 0; + if (rb_memerror_reentered()) { + rb_memerror(); + } rb_bug("object allocation during garbage collection phase"); } @@ -2524,33 +2629,33 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_new } } - obj = newobj_alloc(objspace, cache, size_pool_idx, true); + obj = newobj_alloc(objspace, cache, heap_idx, true); newobj_init(klass, flags, wb_protected, objspace, obj); } - rb_gc_cr_unlock(lev); + RB_GC_CR_UNLOCK(lev); return obj; } NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, - rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx)); + rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx)); NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, - rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx)); + rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx)); static VALUE -newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx) +newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx) { - return newobj_slowpath(klass, flags, objspace, cache, TRUE, size_pool_idx); + return newobj_slowpath(klass, flags, objspace, cache, TRUE, heap_idx); } static VALUE -newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx) +newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx) { - return newobj_slowpath(klass, flags, objspace, cache, FALSE, size_pool_idx); + return newobj_slowpath(klass, flags, objspace, cache, FALSE, heap_idx); } VALUE -rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size) +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) { VALUE obj; rb_objspace_t *objspace = objspace_ptr; @@ -2559,30 +2664,29 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected); if (RB_UNLIKELY(stress_to_class)) { - long cnt = RARRAY_LEN(stress_to_class); - for (long i = 0; i < cnt; i++) { - if (klass == RARRAY_AREF(stress_to_class, i)) rb_memerror(); + if (rb_hash_lookup2(stress_to_class, klass, Qundef) != Qundef) { + rb_memerror(); } } - size_t size_pool_idx = size_pool_idx_for_size(alloc_size); + size_t heap_idx = heap_idx_for_size(alloc_size); rb_ractor_newobj_cache_t *cache = (rb_ractor_newobj_cache_t *)cache_ptr; if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) && wb_protected) { - obj = newobj_alloc(objspace, cache, size_pool_idx, false); + obj = newobj_alloc(objspace, cache, heap_idx, false); newobj_init(klass, flags, wb_protected, objspace, obj); } else { RB_DEBUG_COUNTER_INC(obj_newobj_slowpath); obj = wb_protected ? - newobj_slowpath_wb_protected(klass, flags, objspace, cache, size_pool_idx) : - newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, size_pool_idx); + newobj_slowpath_wb_protected(klass, flags, objspace, cache, heap_idx) : + newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, heap_idx); } - return newobj_fill(obj, v1, v2, v3); + return obj; } static int @@ -2634,7 +2738,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr) if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE; RB_DEBUG_COUNTER_INC(gc_isptr_range); - if (p % BASE_SLOT_SIZE != 0) return FALSE; + if (p % sizeof(VALUE) != 0) return FALSE; RB_DEBUG_COUNTER_INC(gc_isptr_align); page = heap_page_for_ptr(objspace, (uintptr_t)ptr); @@ -2646,7 +2750,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr) else { if (p < page->start) return FALSE; if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE; - if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0) return FALSE; + if ((p - page->start) % page->slot_size != 0) return FALSE; return TRUE; } @@ -2660,7 +2764,7 @@ rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) return is_pointer_to_heap(objspace_ptr, ptr); } -#define ZOMBIE_OBJ_KEPT_FLAGS (FL_SEEN_OBJ_ID | FL_FINALIZE) +#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE) void rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) @@ -2668,7 +2772,7 @@ rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), voi rb_objspace_t *objspace = objspace_ptr; struct RZombie *zombie = RZOMBIE(obj); - zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & ZOMBIE_OBJ_KEPT_FLAGS); + zombie->flags = T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS); zombie->dfree = dfree; zombie->data = data; VALUE prev, next = heap_pages_deferred_final; @@ -2679,24 +2783,7 @@ rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), voi struct heap_page *page = GET_HEAP_PAGE(obj); page->final_slots++; - page->size_pool->final_slots_count++; -} - -static void -obj_free_object_id(rb_objspace_t *objspace, VALUE obj) -{ - st_data_t o = (st_data_t)obj, id; - - GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE || FL_TEST(obj, FL_SEEN_OBJ_ID)); - FL_UNSET(obj, FL_SEEN_OBJ_ID); - - if (st_delete(objspace->obj_to_id_tbl, &o, &id)) { - GC_ASSERT(id); - st_delete(objspace->id_to_obj_tbl, &id, NULL); - } - else { - rb_bug("Object ID seen, but not in mapping table: %s", rb_obj_info(obj)); - } + page->heap->final_slots_count++; } typedef int each_obj_callback(void *, void *, size_t, void *); @@ -2710,8 +2797,8 @@ struct each_obj_data { each_page_callback *each_page_callback; void *data; - struct heap_page **pages[SIZE_POOL_COUNT]; - size_t pages_counts[SIZE_POOL_COUNT]; + struct heap_page **pages[HEAP_COUNT]; + size_t pages_counts[HEAP_COUNT]; }; static VALUE @@ -2725,7 +2812,7 @@ objspace_each_objects_ensure(VALUE arg) objspace->flags.dont_incremental = FALSE; } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { + for (int i = 0; i < HEAP_COUNT; i++) { struct heap_page **pages = data->pages[i]; free(pages); } @@ -2739,10 +2826,10 @@ objspace_each_objects_try(VALUE arg) struct each_obj_data *data = (struct each_obj_data *)arg; rb_objspace_t *objspace = data->objspace; - /* Copy pages from all size_pools to their respective buffers. */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_t size = SIZE_POOL_EDEN_HEAP(size_pool)->total_pages * sizeof(struct heap_page *); + /* Copy pages from all heaps to their respective buffers. */ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + size_t size = heap->total_pages * sizeof(struct heap_page *); struct heap_page **pages = malloc(size); if (!pages) rb_memerror(); @@ -2754,21 +2841,21 @@ objspace_each_objects_try(VALUE arg) * an infinite loop. */ struct heap_page *page = 0; size_t pages_count = 0; - ccan_list_for_each(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node) { + ccan_list_for_each(&heap->pages, page, page_node) { pages[pages_count] = page; pages_count++; } data->pages[i] = pages; data->pages_counts[i] = pages_count; - GC_ASSERT(pages_count == SIZE_POOL_EDEN_HEAP(size_pool)->total_pages); + GC_ASSERT(pages_count == heap->total_pages); } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; size_t pages_count = data->pages_counts[i]; struct heap_page **pages = data->pages[i]; - struct heap_page *page = ccan_list_top(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, struct heap_page, page_node); + struct heap_page *page = ccan_list_top(&heap->pages, struct heap_page, page_node); for (size_t i = 0; i < pages_count; i++) { /* If we have reached the end of the linked list then there are no * more pages, so break. */ @@ -2779,10 +2866,10 @@ objspace_each_objects_try(VALUE arg) if (pages[i] != page) continue; uintptr_t pstart = (uintptr_t)page->start; - uintptr_t pend = pstart + (page->total_slots * size_pool->slot_size); + uintptr_t pend = pstart + (page->total_slots * heap->slot_size); if (data->each_obj_callback && - (*data->each_obj_callback)((void *)pstart, (void *)pend, size_pool->slot_size, data->data)) { + (*data->each_obj_callback)((void *)pstart, (void *)pend, heap->slot_size, data->data)) { break; } if (data->each_page_callback && @@ -2790,7 +2877,7 @@ objspace_each_objects_try(VALUE arg) break; } - page = ccan_list_next(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node); + page = ccan_list_next(&heap->pages, page, page_node); } } @@ -2860,30 +2947,38 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) RBASIC(obj)->flags |= FL_FINALIZE; + unsigned int lev = RB_GC_VM_LOCK(); + if (st_lookup(finalizer_table, obj, &data)) { table = (VALUE)data; + VALUE dup_table = rb_ary_dup(table); + RB_GC_VM_UNLOCK(lev); /* avoid duplicate block, table is usually small */ { long len = RARRAY_LEN(table); long i; for (i = 0; i < len; i++) { - VALUE recv = RARRAY_AREF(table, i); - if (rb_equal(recv, block)) { + VALUE recv = RARRAY_AREF(dup_table, i); + if (rb_equal(recv, block)) { // can't be called with VM lock held return recv; } } } + lev = RB_GC_VM_LOCK(); + RB_GC_GUARD(dup_table); rb_ary_push(table, block); } else { - table = rb_ary_new3(1, block); + table = rb_ary_new3(2, rb_obj_id(obj), block); rb_obj_hide(table); st_add_direct(finalizer_table, obj, table); } + RB_GC_VM_UNLOCK(lev); + return block; } @@ -2895,7 +2990,11 @@ rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) GC_ASSERT(!OBJ_FROZEN(obj)); st_data_t data = obj; + + int lev = RB_GC_VM_LOCK(); st_delete(finalizer_table, &data, 0); + RB_GC_VM_UNLOCK(lev); + FL_UNSET(obj, FL_FINALIZE); } @@ -2908,27 +3007,17 @@ rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) if (!FL_TEST(obj, FL_FINALIZE)) return; + int lev = RB_GC_VM_LOCK(); if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) { - table = (VALUE)data; + table = rb_ary_dup((VALUE)data); + RARRAY_ASET(table, 0, rb_obj_id(dest)); st_insert(finalizer_table, dest, table); FL_SET(dest, FL_FINALIZE); } else { rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj)); } -} - -static VALUE -get_object_id_in_finalizer(rb_objspace_t *objspace, VALUE obj) -{ - if (FL_TEST(obj, FL_SEEN_OBJ_ID)) { - return rb_gc_impl_object_id(objspace, obj); - } - else { - VALUE id = ULL2NUM(objspace->next_object_id); - objspace->next_object_id += OBJ_ID_INCREMENT; - return id; - } + RB_GC_VM_UNLOCK(lev); } static VALUE @@ -2936,11 +3025,11 @@ get_final(long i, void *data) { VALUE table = (VALUE)data; - return RARRAY_AREF(table, i); + return RARRAY_AREF(table, i + 1); } -static void -run_final(rb_objspace_t *objspace, VALUE zombie) +static unsigned int +run_final(rb_objspace_t *objspace, VALUE zombie, unsigned int lev) { if (RZOMBIE(zombie)->dfree) { RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data); @@ -2951,7 +3040,9 @@ run_final(rb_objspace_t *objspace, VALUE zombie) FL_UNSET(zombie, FL_FINALIZE); st_data_t table; if (st_delete(finalizer_table, &key, &table)) { - rb_gc_run_obj_finalizer(get_object_id_in_finalizer(objspace, zombie), RARRAY_LEN(table), get_final, (void *)table); + RB_GC_VM_UNLOCK(lev); + rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table); + lev = RB_GC_VM_LOCK(); } else { rb_bug("FL_FINALIZE flag is set, but finalizers are not found"); @@ -2960,6 +3051,7 @@ run_final(rb_objspace_t *objspace, VALUE zombie) else { GC_ASSERT(!st_lookup(finalizer_table, key, NULL)); } + return lev; } static void @@ -2968,29 +3060,26 @@ finalize_list(rb_objspace_t *objspace, VALUE zombie) while (zombie) { VALUE next_zombie; struct heap_page *page; - asan_unpoison_object(zombie, false); + rb_asan_unpoison_object(zombie, false); next_zombie = RZOMBIE(zombie)->next; page = GET_HEAP_PAGE(zombie); - run_final(objspace, zombie); + unsigned int lev = RB_GC_VM_LOCK(); - int lev = rb_gc_vm_lock(); + lev = run_final(objspace, zombie, lev); { GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE); - if (FL_TEST(zombie, FL_SEEN_OBJ_ID)) { - obj_free_object_id(objspace, zombie); - } - - GC_ASSERT(page->size_pool->final_slots_count > 0); + GC_ASSERT(page->heap->final_slots_count > 0); GC_ASSERT(page->final_slots > 0); - page->size_pool->final_slots_count--; + page->heap->final_slots_count--; page->final_slots--; page->free_slots++; + RVALUE_AGE_SET_BITMAP(zombie, 0); heap_page_add_freeobj(objspace, page, zombie); - page->size_pool->total_freed_objects++; + page->heap->total_freed_objects++; } - rb_gc_vm_unlock(lev); + RB_GC_VM_UNLOCK(lev); zombie = next_zombie; } @@ -3046,9 +3135,9 @@ gc_abort(void *objspace_ptr) } if (is_lazy_sweeping(objspace)) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + objspace->sweeping_heap_count = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; heap->sweeping_page = NULL; struct heap_page *page = NULL; @@ -3059,9 +3148,8 @@ gc_abort(void *objspace_ptr) } } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; rgengc_mark_and_rememberset_clear(objspace, heap); } @@ -3083,7 +3171,10 @@ rb_gc_impl_shutdown_free_objects(void *objspace_ptr) VALUE vp = (VALUE)p; asan_unpoisoning_object(vp) { if (RB_BUILTIN_TYPE(vp) != T_NONE) { - rb_gc_obj_free(objspace, vp); + rb_gc_obj_free_vm_weak_references(vp); + if (rb_gc_obj_free(objspace, vp)) { + RBASIC(vp)->flags = 0; + } } } } @@ -3091,16 +3182,15 @@ rb_gc_impl_shutdown_free_objects(void *objspace_ptr) } static int -rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t data) +rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data) { - rb_objspace_t *objspace = (rb_objspace_t *)data; VALUE obj = (VALUE)key; VALUE table = (VALUE)val; GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE)); GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY); - rb_gc_run_obj_finalizer(rb_gc_impl_object_id(objspace, obj), RARRAY_LEN(table), get_final, (void *)table); + rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table); FL_UNSET(obj, FL_FINALIZE); @@ -3127,7 +3217,7 @@ rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) } while (finalizer_table->num_entries) { - st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, (st_data_t)objspace); + st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0); } /* run finalizers */ @@ -3153,15 +3243,13 @@ rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) uintptr_t pend = p + page->total_slots * stride; for (; p < pend; p += stride) { VALUE vp = (VALUE)p; - void *poisoned = asan_unpoison_object_temporary(vp); - - if (rb_gc_shutdown_call_finalizer_p(vp)) { - rb_gc_obj_free(objspace, vp); - } - - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(vp) == T_NONE); - asan_poison_object(vp); + asan_unpoisoning_object(vp) { + if (rb_gc_shutdown_call_finalizer_p(vp)) { + rb_gc_obj_free_vm_weak_references(vp); + if (rb_gc_obj_free(objspace, vp)) { + RBASIC(vp)->flags = 0; + } + } } } } @@ -3189,13 +3277,8 @@ rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), for (; p < pend; p += stride) { VALUE obj = (VALUE)p; - void *poisoned = asan_unpoison_object_temporary(obj); - - func(obj, data); - - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE); - asan_poison_object(obj); + asan_unpoisoning_object(obj) { + func(obj, data); } } } @@ -3211,9 +3294,9 @@ static size_t objspace_available_slots(rb_objspace_t *objspace) { size_t total_slots = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - total_slots += SIZE_POOL_EDEN_HEAP(size_pool)->total_slots; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + total_slots += heap->total_slots; } return total_slots; } @@ -3238,7 +3321,7 @@ gc_setup_mark_bits(struct heap_page *page) } static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj); -static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size); +static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct heap_page *src_page, struct heap_page *dest_page); #if defined(_WIN32) enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE}; @@ -3249,6 +3332,10 @@ protect_page_body(struct heap_page_body *body, DWORD protect) DWORD old_protect; return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0; } +#elif defined(__wasi__) +// wasi-libc's mprotect emulation does not support PROT_NONE +enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK}; +#define protect_page_body(body, protect) 1 #else enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE}; #define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect)) @@ -3294,8 +3381,10 @@ try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, asan_unlock_freelist(free_page); VALUE dest = (VALUE)free_page->freelist; asan_lock_freelist(free_page); - asan_unpoison_object(dest, false); - if (!dest) { + if (dest) { + rb_asan_unpoison_object(dest, false); + } + else { /* if we can't get something from the freelist then the page must be * full */ return false; @@ -3315,7 +3404,7 @@ try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++; objspace->rcompactor.total_moved++; - gc_move(objspace, src, dest, src_page->slot_size, free_page->slot_size); + gc_move(objspace, src, dest, src_page, free_page); gc_pin(objspace, src); free_page->free_slots--; @@ -3348,33 +3437,27 @@ static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *pag #if GC_CAN_COMPILE_COMPACTION static void -read_barrier_handler(uintptr_t original_address) +read_barrier_handler(uintptr_t address) { - VALUE obj; rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace(); - /* Calculate address aligned to slots. */ - uintptr_t address = original_address - (original_address % BASE_SLOT_SIZE); - - obj = (VALUE)address; - - struct heap_page_body *page_body = GET_PAGE_BODY(obj); + struct heap_page_body *page_body = GET_PAGE_BODY(address); /* If the page_body is NULL, then mprotect cannot handle it and will crash * with "Cannot allocate memory". */ if (page_body == NULL) { - rb_bug("read_barrier_handler: segmentation fault at %p", (void *)original_address); + rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address); } - int lev = rb_gc_vm_lock(); + int lev = RB_GC_VM_LOCK(); { unlock_page_body(objspace, page_body); objspace->profile.read_barrier_faults++; - invalidate_moved_page(objspace, GET_HEAP_PAGE(obj)); + invalidate_moved_page(objspace, GET_HEAP_PAGE(address)); } - rb_gc_vm_unlock(lev); + RB_GC_VM_UNLOCK(lev); } #endif @@ -3523,9 +3606,8 @@ install_handlers(void) static void gc_compact_finish(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; gc_unprotect_pages(objspace, heap); } @@ -3534,9 +3616,8 @@ gc_compact_finish(rb_objspace_t *objspace) gc_update_references(objspace); objspace->profile.compact_count++; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; heap->compact_cursor = NULL; heap->free_pages = NULL; heap->compact_cursor_index = 0; @@ -3561,18 +3642,35 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit { struct heap_page *sweep_page = ctx->page; short slot_size = sweep_page->slot_size; - short slot_bits = slot_size / BASE_SLOT_SIZE; - GC_ASSERT(slot_bits > 0); do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % BASE_SLOT_SIZE == 0); + GC_ASSERT(vp % sizeof(VALUE) == 0); - asan_unpoison_object(vp, false); + rb_asan_unpoison_object(vp, false); if (bitset & 1) { switch (BUILTIN_TYPE(vp)) { - default: /* majority case */ - gc_report(2, objspace, "page_sweep: free %p\n", (void *)p); + case T_MOVED: + if (objspace->flags.during_compacting) { + /* The sweep cursor shouldn't have made it to any + * T_MOVED slots while the compact flag is enabled. + * The sweep cursor and compact cursor move in + * opposite directions, and when they meet references will + * get updated and "during_compacting" should get disabled */ + rb_bug("T_MOVED shouldn't be seen until compaction is finished"); + } + gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); + ctx->empty_slots++; + heap_page_add_freeobj(objspace, sweep_page, vp); + break; + case T_ZOMBIE: + /* already counted */ + break; + case T_NONE: + ctx->empty_slots++; /* already freed */ + break; + + default: #if RGENGC_CHECK_MODE if (!is_full_marking(objspace)) { if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p); @@ -3580,8 +3678,6 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit } #endif - if (RVALUE_WB_UNPROTECTED(objspace, vp)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(vp), vp); - #if RGENGC_CHECK_MODE #define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp)) CHECK(RVALUE_WB_UNPROTECTED); @@ -3591,46 +3687,31 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit #undef CHECK #endif - bool has_object_id = FL_TEST(vp, FL_SEEN_OBJ_ID); - if (rb_gc_obj_free(objspace, vp)) { - if (has_object_id) { - obj_free_object_id(objspace, vp); - } - // always add free slots back to the swept pages freelist, - // so that if we're compacting, we can re-use the slots - (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE); + if (!rb_gc_obj_needs_cleanup_p(vp)) { + (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size); heap_page_add_freeobj(objspace, sweep_page, vp); - gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); + gc_report(3, objspace, "page_sweep: %s (fast path) added to freelist\n", rb_obj_info(vp)); ctx->freed_slots++; } else { - ctx->final_slots++; - } - break; - - case T_MOVED: - if (objspace->flags.during_compacting) { - /* The sweep cursor shouldn't have made it to any - * T_MOVED slots while the compact flag is enabled. - * The sweep cursor and compact cursor move in - * opposite directions, and when they meet references will - * get updated and "during_compacting" should get disabled */ - rb_bug("T_MOVED shouldn't be seen until compaction is finished"); + gc_report(2, objspace, "page_sweep: free %p\n", (void *)p); + + rb_gc_obj_free_vm_weak_references(vp); + if (rb_gc_obj_free(objspace, vp)) { + (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size); + heap_page_add_freeobj(objspace, sweep_page, vp); + gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); + ctx->freed_slots++; + } + else { + ctx->final_slots++; + } } - gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); - ctx->empty_slots++; - heap_page_add_freeobj(objspace, sweep_page, vp); - break; - case T_ZOMBIE: - /* already counted */ - break; - case T_NONE: - ctx->empty_slots++; /* already freed */ break; } } p += slot_size; - bitset >>= slot_bits; + bitset >>= 1; } while (bitset); } @@ -3638,7 +3719,7 @@ static inline void gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx) { struct heap_page *sweep_page = ctx->page; - GC_ASSERT(SIZE_POOL_EDEN_HEAP(sweep_page->size_pool) == heap); + GC_ASSERT(sweep_page->heap == heap); uintptr_t p; bits_t *bits, bitset; @@ -3655,34 +3736,33 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context p = (uintptr_t)sweep_page->start; bits = sweep_page->mark_bits; + short slot_size = sweep_page->slot_size; + int total_slots = sweep_page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); - int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE); - int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH; - if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64 - bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1); + int out_of_range_bits = total_slots % BITS_BITLENGTH; + if (out_of_range_bits != 0) { + bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1); } - /* The last bitmap plane may not be used if the last plane does not - * have enough space for the slot_size. In that case, the last plane must - * be skipped since none of the bits will be set. */ - int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH); - GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 || - bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT); - - // Skip out of range slots at the head of the page - bitset = ~bits[0]; - bitset >>= NUM_IN_PAGE(p); - if (bitset) { - gc_sweep_plane(objspace, heap, p, bitset, ctx); + // Clear wb_unprotected and age bits for all unmarked slots + { + bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits; + bits_t *age_bits = sweep_page->age_bits; + for (int i = 0; i < bitmap_plane_count; i++) { + bits_t unmarked = ~bits[i]; + wb_unprotected_bits[i] &= ~unmarked; + age_bits[i * 2] &= ~unmarked; + age_bits[i * 2 + 1] &= ~unmarked; + } } - p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; - for (int i = 1; i < bitmap_plane_count; i++) { + for (int i = 0; i < bitmap_plane_count; i++) { bitset = ~bits[i]; if (bitset) { gc_sweep_plane(objspace, heap, p, bitset, ctx); } - p += BITS_BITLENGTH * BASE_SLOT_SIZE; + p += BITS_BITLENGTH * slot_size; } if (!heap->compact_cursor) { @@ -3702,7 +3782,7 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context ctx->freed_slots, ctx->empty_slots, ctx->final_slots); sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots; - sweep_page->size_pool->total_freed_objects += ctx->freed_slots; + sweep_page->heap->total_freed_objects += ctx->freed_slots; if (heap_pages_deferred_final && !finalizing) { gc_finalize_deferred_register(objspace); @@ -3714,7 +3794,10 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context struct free_slot *ptr = sweep_page->freelist; while (ptr) { freelist_len++; - ptr = ptr->next; + rb_asan_unpoison_object((VALUE)ptr, false); + struct free_slot *next = ptr->next; + rb_asan_poison_object((VALUE)ptr); + ptr = next; } asan_lock_freelist(sweep_page); if (freelist_len != sweep_page->free_slots) { @@ -3760,15 +3843,15 @@ heap_page_freelist_append(struct heap_page *page, struct free_slot *freelist) asan_unlock_freelist(page); if (page->freelist) { struct free_slot *p = page->freelist; - asan_unpoison_object((VALUE)p, false); + rb_asan_unpoison_object((VALUE)p, false); while (p->next) { struct free_slot *prev = p; p = p->next; - asan_poison_object((VALUE)prev); - asan_unpoison_object((VALUE)p, false); + rb_asan_poison_object((VALUE)prev); + rb_asan_unpoison_object((VALUE)p, false); } p->next = freelist; - asan_poison_object((VALUE)p); + rb_asan_poison_object((VALUE)p); } else { page->freelist = freelist; @@ -3781,6 +3864,9 @@ static void gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap) { heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node); + if (heap->sweeping_page) { + objspace->sweeping_heap_count++; + } heap->free_pages = NULL; heap->pooled_pages = NULL; if (!objspace->flags.immediate_sweep) { @@ -3804,12 +3890,18 @@ static int compare_pinned_slots(const void *left, const void *right, void *d); static void gc_ractor_newobj_cache_clear(void *c, void *data) { + rb_objspace_t *objspace = rb_gc_get_objspace(); rb_ractor_newobj_cache_t *newobj_cache = c; newobj_cache->incremental_mark_step_allocated_slots = 0; - for (size_t size_pool_idx = 0; size_pool_idx < SIZE_POOL_COUNT; size_pool_idx++) { - rb_ractor_newobj_size_pool_cache_t *cache = &newobj_cache->size_pool_caches[size_pool_idx]; + for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) { + + rb_ractor_newobj_heap_cache_t *cache = &newobj_cache->heap_caches[heap_idx]; + + rb_heap_t *heap = &heaps[heap_idx]; + RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, cache->allocated_objects_count); + cache->allocated_objects_count = 0; struct heap_page *page = cache->using_page; struct free_slot *freelist = cache->freelist; @@ -3823,11 +3915,70 @@ gc_ractor_newobj_cache_clear(void *c, void *data) } static void +gc_sweep_freeobj_hooks_page(rb_objspace_t *objspace, struct heap_page *page) +{ + bits_t *bits = page->mark_bits; + uintptr_t p = (uintptr_t)page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); + + int out_of_range_bits = total_slots % BITS_BITLENGTH; + bits_t last_plane_mask = (out_of_range_bits != 0) + ? ~(((bits_t)1 << out_of_range_bits) - 1) + : 0; + + for (int j = 0; j < bitmap_plane_count; j++) { + bits_t bitset = ~bits[j]; + if (j == bitmap_plane_count - 1) { + bitset &= ~last_plane_mask; + } + + uintptr_t pp = p; + while (bitset) { + if (bitset & 1) { + VALUE vp = (VALUE)pp; + asan_unpoisoning_object(vp) { + switch (BUILTIN_TYPE(vp)) { + case T_NONE: + case T_ZOMBIE: + case T_MOVED: + break; + default: + rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ); + break; + } + } + } + pp += slot_size; + bitset >>= 1; + } + p += BITS_BITLENGTH * slot_size; + } +} + +static void +gc_sweep_freeobj_hooks(rb_objspace_t *objspace) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + struct heap_page *page = NULL; + + ccan_list_for_each(&heap->pages, page, page_node) { + gc_sweep_freeobj_hooks_page(objspace, page); + } + } +} + +static void gc_sweep_start(rb_objspace_t *objspace) { gc_mode_transition(objspace, gc_mode_sweeping); objspace->rincgc.pooled_slots = 0; - objspace->heap_pages.allocatable_slots = 0; + + if (RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_FREEOBJ)) { + gc_sweep_freeobj_hooks(objspace); + } #if GC_CAN_COMPILE_COMPACTION if (objspace->flags.during_compacting) { @@ -3838,17 +3989,15 @@ gc_sweep_start(rb_objspace_t *objspace) } #endif - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); - + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; gc_sweep_start_heap(objspace, heap); - /* We should call gc_sweep_finish_size_pool for size pools with no pages. */ + /* We should call gc_sweep_finish_heap for size pools with no pages. */ if (heap->sweeping_page == NULL) { GC_ASSERT(heap->total_pages == 0); GC_ASSERT(heap->total_slots == 0); - gc_sweep_finish_size_pool(objspace, size_pool); + gc_sweep_finish_heap(objspace, heap); } } @@ -3856,18 +4005,17 @@ gc_sweep_start(rb_objspace_t *objspace) } static void -gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) +gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap) { - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); size_t total_slots = heap->total_slots; - size_t swept_slots = size_pool->freed_slots + size_pool->empty_slots; + size_t swept_slots = heap->freed_slots + heap->empty_slots; - size_t init_slots = gc_params.size_pool_init_slots[size_pool - size_pools]; + size_t init_slots = gc_params.heap_init_bytes / heap->slot_size; size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio); if (swept_slots < min_free_slots && /* The heap is a growth heap if it freed more slots than had empty slots. */ - (size_pool->empty_slots == 0 || size_pool->freed_slots > size_pool->empty_slots)) { + ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) { /* If we don't have enough slots and we have pages on the tomb heap, move * pages from the tomb heap to the eden heap. This may prevent page * creation thrashing (frequently allocating and deallocting pages) and @@ -3875,7 +4023,7 @@ gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) struct heap_page *resurrected_page; while (swept_slots < min_free_slots && (resurrected_page = heap_page_resurrect(objspace))) { - size_pool_add_page(objspace, size_pool, heap, resurrected_page); + heap_add_page(objspace, heap, resurrected_page); heap_add_freepage(heap, resurrected_page); swept_slots += resurrected_page->free_slots; @@ -3883,14 +4031,17 @@ gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) if (swept_slots < min_free_slots) { /* Grow this heap if we are in a major GC or if we haven't run at least - * RVALUE_OLD_AGE minor GC since the last major GC. */ + * RVALUE_OLD_AGE minor GC since the last major GC. */ if (is_full_marking(objspace) || objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) { - size_pool_allocatable_slots_expand(objspace, size_pool, swept_slots, heap->total_slots); + if (objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) { + heap_allocatable_bytes_expand(objspace, heap, swept_slots, heap->total_slots, heap->slot_size); + } } - else { + else if (swept_slots < min_free_slots * 7 / 8 && + objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) { gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE; - size_pool->force_major_gc_count++; + heap->force_major_gc_count++; } } } @@ -3904,27 +4055,33 @@ gc_sweep_finish(rb_objspace_t *objspace) gc_prof_set_heap_info(objspace); heap_pages_free_unused_pages(objspace); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; - size_pool->freed_slots = 0; - size_pool->empty_slots = 0; + heap->freed_slots = 0; + heap->empty_slots = 0; if (!will_be_incremental_marking(objspace)) { - rb_heap_t *eden_heap = SIZE_POOL_EDEN_HEAP(size_pool); - struct heap_page *end_page = eden_heap->free_pages; + struct heap_page *end_page = heap->free_pages; if (end_page) { while (end_page->free_next) end_page = end_page->free_next; - end_page->free_next = eden_heap->pooled_pages; + end_page->free_next = heap->pooled_pages; } else { - eden_heap->free_pages = eden_heap->pooled_pages; + heap->free_pages = heap->pooled_pages; } - eden_heap->pooled_pages = NULL; + heap->pooled_pages = NULL; objspace->rincgc.pooled_slots = 0; } } + (void)gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters); +#if RGENGC_ESTIMATE_OLDMALLOC + if (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) { + (void)gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.oldcounters); + } +#endif + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP); gc_mode_transition(objspace, gc_mode_none); @@ -3934,12 +4091,13 @@ gc_sweep_finish(rb_objspace_t *objspace) } static int -gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap) { struct heap_page *sweep_page = heap->sweeping_page; - int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP; int swept_slots = 0; int pooled_slots = 0; + int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size; + int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size; if (sweep_page == NULL) return FALSE; @@ -3961,40 +4119,38 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *hea heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node); - if (free_slots == sweep_page->total_slots && - heap_pages_freeable_pages > 0 && - unlink_limit > 0) { - heap_pages_freeable_pages--; - unlink_limit--; + if (free_slots == sweep_page->total_slots) { /* There are no living objects, so move this page to the global empty pages. */ heap_unlink_page(objspace, heap, sweep_page); sweep_page->start = 0; sweep_page->total_slots = 0; sweep_page->slot_size = 0; - sweep_page->size_pool = NULL; + sweep_page->heap = NULL; sweep_page->free_slots = 0; asan_unlock_freelist(sweep_page); sweep_page->freelist = NULL; asan_lock_freelist(sweep_page); + asan_poison_memory_region(sweep_page->body, HEAP_PAGE_SIZE); + objspace->empty_pages_count++; sweep_page->free_next = objspace->empty_pages; objspace->empty_pages = sweep_page; } else if (free_slots > 0) { - size_pool->freed_slots += ctx.freed_slots; - size_pool->empty_slots += ctx.empty_slots; + heap->freed_slots += ctx.freed_slots; + heap->empty_slots += ctx.empty_slots; - if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) { + if (pooled_slots < pool_budget) { heap_add_poolpage(objspace, heap, sweep_page); pooled_slots += free_slots; } else { heap_add_freepage(heap, sweep_page); swept_slots += free_slots; - if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) { + if (swept_slots > sweep_budget) { break; } } @@ -4005,7 +4161,9 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *hea } while ((sweep_page = heap->sweeping_page)); if (!heap->sweeping_page) { - gc_sweep_finish_size_pool(objspace, size_pool); + objspace->sweeping_heap_count--; + GC_ASSERT(objspace->sweeping_heap_count >= 0); + gc_sweep_finish_heap(objspace, heap); if (!has_sweeping_pages(objspace)) { gc_sweep_finish(objspace); @@ -4022,31 +4180,48 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *hea static void gc_sweep_rest(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; - while (SIZE_POOL_EDEN_HEAP(size_pool)->sweeping_page) { - gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool)); + while (heap->sweeping_page) { + gc_sweep_step(objspace, heap); } } } static void -gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *sweep_size_pool, rb_heap_t *heap) +gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap) { GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD); if (!GC_ENABLE_LAZY_SWEEP) return; gc_sweeping_enter(objspace); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - if (!gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool))) { - /* sweep_size_pool requires a free slot but sweeping did not yield any - * and we cannot allocate a new page. */ - if (size_pool == sweep_size_pool && objspace->heap_pages.allocatable_slots == 0) { + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + if (gc_sweep_step(objspace, heap)) { + GC_ASSERT(heap->free_pages != NULL); + } + else if (heap == sweep_heap) { + if (objspace->empty_pages_count > 0 || objspace->heap_pages.allocatable_bytes > 0) { + /* [Bug #21548] + * + * If this heap is the heap we want to sweep, but we weren't able + * to free any slots, but we also either have empty pages or could + * allocate new pages, then we want to preemptively claim a page + * because it's possible that sweeping another heap will call + * gc_sweep_finish_heap, which may use up all of the + * empty/allocatable pages. If other heaps are not finished sweeping + * then we do not finish this GC and we will end up triggering a new + * GC cycle during this GC phase. */ + heap_page_allocate_and_initialize(objspace, heap); + + GC_ASSERT(heap->free_pages != NULL); + } + else { /* Not allowed to create a new page so finish sweeping. */ gc_sweep_rest(objspace); + GC_ASSERT(gc_mode(objspace) == gc_mode_none); break; } } @@ -4060,9 +4235,7 @@ rb_gc_impl_location(void *objspace_ptr, VALUE value) { VALUE destination; - if (!SPECIAL_CONST_P(value)) { - void *poisoned = asan_unpoison_object_temporary(value); - + asan_unpoisoning_object(value) { if (BUILTIN_TYPE(value) == T_MOVED) { destination = (VALUE)RMOVED(value)->destination; GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE); @@ -4070,15 +4243,6 @@ rb_gc_impl_location(void *objspace_ptr, VALUE value) else { destination = value; } - - /* Re-poison slot if it's not the one we want */ - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(value) == T_NONE); - asan_poison_object(value); - } - } - else { - destination = value; } return destination; @@ -4101,22 +4265,13 @@ invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_ CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object); object = rb_gc_impl_location(objspace, forwarding_object); - - uint32_t original_shape_id = 0; - if (RB_TYPE_P(object, T_OBJECT)) { - original_shape_id = RMOVED(forwarding_object)->original_shape_id; - } - - gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object)->slot_size, page->slot_size); + gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object), page); /* forwarding_object is now our actual object, and "object" * is the free slot for the original page */ - if (original_shape_id) { - rb_gc_set_shape(forwarding_object, original_shape_id); - } - struct heap_page *orig_page = GET_HEAP_PAGE(object); orig_page->free_slots++; + RVALUE_AGE_SET_BITMAP(object, 0); heap_page_add_freeobj(objspace, orig_page, object); GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object)); @@ -4124,7 +4279,7 @@ invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_ GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE); } } - p += BASE_SLOT_SIZE; + p += page->slot_size; bitset >>= 1; } while (bitset); } @@ -4136,25 +4291,21 @@ invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page) int i; bits_t *mark_bits, *pin_bits; bits_t bitset; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); mark_bits = page->mark_bits; pin_bits = page->pinned_bits; uintptr_t p = page->start; - // Skip out of range slots at the head of the page - bitset = pin_bits[0] & ~mark_bits[0]; - bitset >>= NUM_IN_PAGE(p); - invalidate_moved_plane(objspace, page, p, bitset); - p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; - - for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) { + for (i=0; i < bitmap_plane_count; i++) { /* Moved objects are pinned but never marked. We reuse the pin bits * to indicate there is a moved object in this slot. */ bitset = pin_bits[i] & ~mark_bits[i]; - invalidate_moved_plane(objspace, page, p, bitset); - p += BITS_BITLENGTH * BASE_SLOT_SIZE; + p += BITS_BITLENGTH * slot_size; } } #endif @@ -4165,8 +4316,8 @@ gc_compact_start(rb_objspace_t *objspace) struct heap_page *page = NULL; gc_mode_transition(objspace, gc_mode_compacting); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(&size_pools[i]); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; ccan_list_for_each(&heap->pages, page, page_node) { page->flags.before_sweep = TRUE; } @@ -4217,9 +4368,9 @@ gc_sweep(rb_objspace_t *objspace) else { /* Sweep every size pool. */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool)); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + gc_sweep_step(objspace, heap); } } @@ -4426,15 +4577,11 @@ init_mark_stack(mark_stack_t *stack) static void rgengc_check_relation(rb_objspace_t *objspace, VALUE obj) { - const VALUE old_parent = objspace->rgengc.parent_object; - - if (old_parent) { /* parent object is old */ + if (objspace->rgengc.parent_object_old_p) { if (RVALUE_WB_UNPROTECTED(objspace, obj) || !RVALUE_OLD_P(objspace, obj)) { - rgengc_remember(objspace, old_parent); + rgengc_remember(objspace, objspace->rgengc.parent_object); } } - - GC_ASSERT(old_parent == objspace->rgengc.parent_object); } static inline int @@ -4465,8 +4612,16 @@ gc_aging(rb_objspace_t *objspace, VALUE obj) if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) { if (!RVALUE_OLD_P(objspace, obj)) { - gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj)); - RVALUE_AGE_INC(objspace, obj); + int t = BUILTIN_TYPE(obj); + if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) { + gc_report(3, objspace, "gc_aging: YOUNG class: %s\n", rb_obj_info(obj)); + RVALUE_AGE_SET(obj, RVALUE_OLD_AGE); + RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj); + } + else { + gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj)); + RVALUE_AGE_INC(objspace, obj); + } } else if (is_full_marking(objspace)) { GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE); @@ -4490,32 +4645,44 @@ gc_grey(rb_objspace_t *objspace, VALUE obj) MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj); } + if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) { + rb_darray_append_without_gc(&objspace->weak_references, obj); + } + push_mark_stack(&objspace->mark_stack, obj); } +static inline void +gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj) +{ + if (RB_UNLIKELY(BUILTIN_TYPE(obj) == T_NONE)) { + enum {info_size = 256}; + char obj_info_buf[info_size]; + rb_raw_obj_info(obj_info_buf, info_size, obj); + + char parent_obj_info_buf[info_size]; + rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object); + + rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf); + } +} + static void gc_mark(rb_objspace_t *objspace, VALUE obj) { GC_ASSERT(during_gc); + GC_ASSERT(!objspace->flags.during_reference_updating); rgengc_check_relation(objspace, obj); if (!gc_mark_set(objspace, obj)) return; /* already marked */ if (0) { // for debug GC marking miss - if (objspace->rgengc.parent_object) { - RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)", - (void *)obj, obj_type_name(obj), - (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object)); - } - else { - RUBY_DEBUG_LOG("%p (%s)", (void *)obj, obj_type_name(obj)); - } + RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)", + (void *)obj, obj_type_name(obj), + (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object)); } - if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) { - rb_obj_info_dump(obj); - rb_bug("try to mark T_NONE object"); /* check here will help debugging */ - } + gc_mark_check_t_none(objspace, obj); gc_aging(objspace, obj); gc_grey(objspace, obj); @@ -4552,7 +4719,10 @@ rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) GC_ASSERT(objspace->flags.during_compacting); GC_ASSERT(during_gc); - *ptr = rb_gc_impl_location(objspace, *ptr); + VALUE destination = rb_gc_impl_location(objspace, *ptr); + if (destination != *ptr) { + *ptr = destination; + } } else { gc_mark(objspace, *ptr); @@ -4583,90 +4753,48 @@ rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj)); if (is_pointer_to_heap(objspace, (void *)obj)) { - void *ptr = asan_unpoison_object_temporary(obj); - - /* Garbage can live on the stack, so do not mark or pin */ - switch (BUILTIN_TYPE(obj)) { - case T_ZOMBIE: - case T_NONE: - break; - default: - gc_mark_and_pin(objspace, obj); - break; - } - - if (ptr) { - GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE); - asan_poison_object(obj); + asan_unpoisoning_object(obj) { + /* Garbage can live on the stack, so do not mark or pin */ + switch (BUILTIN_TYPE(obj)) { + case T_ZOMBIE: + case T_NONE: + break; + default: + gc_mark_and_pin(objspace, obj); + break; + } } } } -void -rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr) +static int +pin_value(st_data_t key, st_data_t value, st_data_t data) { - rb_objspace_t *objspace = objspace_ptr; - - if (RB_UNLIKELY(!during_gc)) return; - - VALUE obj = *ptr; - if (RB_SPECIAL_CONST_P(obj)) return; - - GC_ASSERT(objspace->rgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED)); - - if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) { - rb_obj_info_dump(obj); - rb_bug("try to mark T_NONE object"); - } - - /* If we are in a minor GC and the other object is old, then obj should - * already be marked and cannot be reclaimed in this GC cycle so we don't - * need to add it to the weak refences list. */ - if (!is_full_marking(objspace) && RVALUE_OLD_P(objspace, obj)) { - GC_ASSERT(RVALUE_MARKED(objspace, obj)); - GC_ASSERT(!objspace->flags.during_compacting); - - return; - } - - rgengc_check_relation(objspace, obj); - - DURING_GC_COULD_MALLOC_REGION_START(); - { - rb_darray_append(&objspace->weak_references, ptr); - } - DURING_GC_COULD_MALLOC_REGION_END(); + rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); - objspace->profile.weak_references_count++; + return ST_CONTINUE; } -void -rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr) +static inline void +gc_mark_set_parent_raw(rb_objspace_t *objspace, VALUE obj, bool old_p) { - rb_objspace_t *objspace = objspace_ptr; - - /* If we're not incremental marking, then the state of the objects can't - * change so we don't need to do anything. */ - if (!is_incremental_marking(objspace)) return; - /* If parent_obj has not been marked, then ptr has not yet been marked - * weak, so we don't need to do anything. */ - if (!RVALUE_MARKED(objspace, parent_obj)) return; - - VALUE **ptr_ptr; - rb_darray_foreach(objspace->weak_references, i, ptr_ptr) { - if (*ptr_ptr == ptr) { - *ptr_ptr = NULL; - break; - } - } + asan_unpoison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object), false); + asan_unpoison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p), false); + objspace->rgengc.parent_object = obj; + objspace->rgengc.parent_object_old_p = old_p; } -static int -pin_value(st_data_t key, st_data_t value, st_data_t data) +static inline void +gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj) { - rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); + gc_mark_set_parent_raw(objspace, obj, RVALUE_OLD_P(objspace, obj)); +} - return ST_CONTINUE; +static inline void +gc_mark_set_parent_invalid(rb_objspace_t *objspace) +{ + asan_poison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object)); + asan_poison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p)); } static void @@ -4677,28 +4805,17 @@ mark_roots(rb_objspace_t *objspace, const char **categoryp) } while (0) MARK_CHECKPOINT("objspace"); - objspace->rgengc.parent_object = Qfalse; + gc_mark_set_parent_raw(objspace, Qundef, false); if (finalizer_table != NULL) { st_foreach(finalizer_table, pin_value, (st_data_t)objspace); } - st_foreach(objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)objspace); - if (stress_to_class) rb_gc_mark(stress_to_class); + rb_gc_save_machine_context(); rb_gc_mark_roots(objspace, categoryp); -} - -static inline void -gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj) -{ - if (RVALUE_OLD_P(objspace, obj)) { - objspace->rgengc.parent_object = obj; - } - else { - objspace->rgengc.parent_object = Qfalse; - } + gc_mark_set_parent_invalid(objspace); } static void @@ -4706,6 +4823,7 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) { gc_mark_set_parent(objspace, obj); rb_gc_mark_children(objspace, obj); + gc_mark_set_parent_invalid(objspace); } /** @@ -4890,10 +5008,10 @@ allrefs_roots_i(VALUE obj, void *ptr) } } #define PUSH_MARK_FUNC_DATA(v) do { \ - struct gc_mark_func_data_struct *prev_mark_func_data = GET_RACTOR()->mfd; \ - GET_RACTOR()->mfd = (v); + struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \ + GET_VM()->gc.mark_func_data = (v); -#define POP_MARK_FUNC_DATA() GET_RACTOR()->mfd = prev_mark_func_data;} while (0) +#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0) static st_table * objspace_allrefs(rb_objspace_t *objspace) @@ -4913,7 +5031,7 @@ objspace_allrefs(rb_objspace_t *objspace) /* traverse root objects */ PUSH_MARK_FUNC_DATA(&mfd); - GET_RACTOR()->mfd = &mfd; + GET_VM()->gc.mark_func_data = &mfd; mark_roots(objspace, &data.category); POP_MARK_FUNC_DATA(); @@ -4971,7 +5089,7 @@ gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr) rb_objspace_t *objspace = (rb_objspace_t *)ptr; /* object should be marked or oldgen */ - if (!RVALUE_MARKED(obj)) { + if (!RVALUE_MARKED(objspace, obj)) { fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj)); fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj); reflist_dump(refs); @@ -4991,10 +5109,22 @@ gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr) static void gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name) { - size_t saved_malloc_increase = objspace->malloc_params.increase; + MALLOC_COUNTERS_LOCK(objspace); + struct gc_malloc_bytes saved_malloc = { + .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc), + .free = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free), + .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc), + .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc), + }; #if RGENGC_ESTIMATE_OLDMALLOC - size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase; + struct gc_malloc_bytes saved_oldmalloc = { + .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.malloc), + .free = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.free), + .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.malloc_at_last_gc), + .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.free_at_last_gc), + }; #endif + MALLOC_COUNTERS_UNLOCK(objspace); VALUE already_disabled = rb_objspace_gc_disable(objspace); objspace->rgengc.allrefs_table = objspace_allrefs(objspace); @@ -5014,10 +5144,18 @@ gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, objspace->rgengc.allrefs_table = 0; if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace); - objspace->malloc_params.increase = saved_malloc_increase; + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_store_release(&objspace->malloc_counters.counters.malloc, saved_malloc.malloc); + gc_counter_store_release(&objspace->malloc_counters.counters.free, saved_malloc.free); + gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc); + gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc); #if RGENGC_ESTIMATE_OLDMALLOC - objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase; + gc_counter_store_release(&objspace->malloc_counters.oldcounters.malloc, saved_oldmalloc.malloc); + gc_counter_store_release(&objspace->malloc_counters.oldcounters.free, saved_oldmalloc.free); + gc_counter_store_release(&objspace->malloc_counters.oldcounters.malloc_at_last_gc, saved_oldmalloc.malloc_at_last_gc); + gc_counter_store_release(&objspace->malloc_counters.oldcounters.free_at_last_gc, saved_oldmalloc.free_at_last_gc); #endif + MALLOC_COUNTERS_UNLOCK(objspace); } #endif /* RGENGC_CHECK_MODE >= 4 */ @@ -5083,59 +5221,59 @@ verify_internal_consistency_i(void *page_start, void *page_end, size_t stride, rb_objspace_t *objspace = data->objspace; for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) { - void *poisoned = asan_unpoison_object_temporary(obj); - - if (!rb_gc_impl_garbage_object_p(objspace, obj)) { - /* count objects */ - data->live_object_count++; - data->parent = obj; - - /* Normally, we don't expect T_MOVED objects to be in the heap. - * But they can stay alive on the stack, */ - if (!gc_object_moved_p(objspace, obj)) { - /* moved slots don't have children */ - rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data); - } + asan_unpoisoning_object(obj) { + if (!rb_gc_impl_garbage_object_p(objspace, obj)) { + /* count objects */ + data->live_object_count++; + data->parent = obj; - /* check health of children */ - if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++; - if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++; + /* Normally, we don't expect T_MOVED objects to be in the heap. + * But they can stay alive on the stack, */ + if (!gc_object_moved_p(objspace, obj)) { + /* moved slots don't have children */ + rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data); + } - if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) { - /* reachable objects from an oldgen object should be old or (young with remember) */ - data->parent = obj; - rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data); - } + /* check health of children */ + if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++; + if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++; - if (is_incremental_marking(objspace)) { - if (RVALUE_BLACK_P(objspace, obj)) { - /* reachable objects from black objects should be black or grey objects */ + if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) { + /* reachable objects from an oldgen object should be old or (young with remember) */ data->parent = obj; - rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data); + rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data); } - } - } - else { - if (BUILTIN_TYPE(obj) == T_ZOMBIE) { - data->zombie_object_count++; - if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) { - fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n", - rb_obj_info(obj)); - data->err_count++; + if (!is_marking(objspace) && rb_gc_obj_shareable_p(obj)) { + rb_gc_verify_shareable(obj); } - if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) { - fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n", - FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in", - rb_obj_info(obj)); - data->err_count++; + if (is_incremental_marking(objspace)) { + if (RVALUE_BLACK_P(objspace, obj)) { + /* reachable objects from black objects should be black or grey objects */ + data->parent = obj; + rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data); + } + } + } + else { + if (BUILTIN_TYPE(obj) == T_ZOMBIE) { + data->zombie_object_count++; + + if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) { + fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n", + rb_obj_info(obj)); + data->err_count++; + } + + if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) { + fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n", + FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in", + rb_obj_info(obj)); + data->err_count++; + } } } - } - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE); - asan_poison_object(obj); } } @@ -5157,22 +5295,18 @@ gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) for (uintptr_t ptr = start; ptr < end; ptr += slot_size) { VALUE val = (VALUE)ptr; - void *poisoned = asan_unpoison_object_temporary(val); - enum ruby_value_type type = BUILTIN_TYPE(val); + asan_unpoisoning_object(val) { + enum ruby_value_type type = BUILTIN_TYPE(val); - if (type == T_NONE) free_objects++; - if (type == T_ZOMBIE) zombie_objects++; - if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) { - has_remembered_shady = TRUE; - } - if (RVALUE_PAGE_MARKING(page, val)) { - has_remembered_old = TRUE; - remembered_old_objects++; - } - - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(val) == T_NONE); - asan_poison_object(val); + if (type == T_NONE) free_objects++; + if (type == T_ZOMBIE) zombie_objects++; + if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) { + has_remembered_shady = TRUE; + } + if (RVALUE_PAGE_MARKING(page, val)) { + has_remembered_old = TRUE; + remembered_old_objects++; + } } } @@ -5219,12 +5353,12 @@ gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head) while (p) { VALUE vp = (VALUE)p; VALUE prev = vp; - asan_unpoison_object(vp, false); + rb_asan_unpoison_object(vp, false); if (BUILTIN_TYPE(vp) != T_NONE) { fprintf(stderr, "freelist slot expected to be T_NONE but was: %s\n", rb_obj_info(vp)); } p = p->next; - asan_poison_object(prev); + rb_asan_poison_object(prev); } asan_lock_freelist(page); @@ -5240,8 +5374,8 @@ static int gc_verify_heap_pages(rb_objspace_t *objspace) { int remembered_old_objects = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - remembered_old_objects += gc_verify_heap_pages_(objspace, &(SIZE_POOL_EDEN_HEAP(&size_pools[i])->pages)); + for (int i = 0; i < HEAP_COUNT; i++) { + remembered_old_objects += gc_verify_heap_pages_(objspace, &((&heaps[i])->pages)); } return remembered_old_objects; } @@ -5279,6 +5413,8 @@ gc_verify_internal_consistency_(rb_objspace_t *objspace) /* check counters */ + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + if (!is_lazy_sweeping(objspace) && !finalizing && !rb_gc_multi_ractor_p()) { @@ -5333,7 +5469,7 @@ gc_verify_internal_consistency(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - unsigned int lev = rb_gc_vm_lock(); + unsigned int lev = RB_GC_VM_LOCK(); { rb_gc_vm_barrier(); // stop other ractors @@ -5344,7 +5480,7 @@ gc_verify_internal_consistency(void *objspace_ptr) } during_gc = prev_during_gc; } - rb_gc_vm_unlock(lev); + RB_GC_VM_UNLOCK(lev); } static void @@ -5391,7 +5527,7 @@ gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj) } static inline void -gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits) +gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits, short slot_size) { if (bits) { do { @@ -5401,7 +5537,7 @@ gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p)); gc_mark_children(objspace, (VALUE)p); } - p += BASE_SLOT_SIZE; + p += slot_size; bits >>= 1; } while (bits); } @@ -5416,52 +5552,63 @@ gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap) bits_t *mark_bits = page->mark_bits; bits_t *wbun_bits = page->wb_unprotected_bits; uintptr_t p = page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); size_t j; - bits_t bits = mark_bits[0] & wbun_bits[0]; - bits >>= NUM_IN_PAGE(p); - gc_marks_wb_unprotected_objects_plane(objspace, p, bits); - p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; - - for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) { + for (j=0; j<(size_t)bitmap_plane_count; j++) { bits_t bits = mark_bits[j] & wbun_bits[j]; - - gc_marks_wb_unprotected_objects_plane(objspace, p, bits); - p += BITS_BITLENGTH * BASE_SLOT_SIZE; + gc_marks_wb_unprotected_objects_plane(objspace, p, bits, slot_size); + p += BITS_BITLENGTH * slot_size; } } gc_mark_stacked_objects_all(objspace); } -static void -gc_update_weak_references(rb_objspace_t *objspace) +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) { - size_t retained_weak_references_count = 0; - VALUE **ptr_ptr; - rb_darray_foreach(objspace->weak_references, i, ptr_ptr) { - if (!*ptr_ptr) continue; + FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE); +} - VALUE obj = **ptr_ptr; +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; - if (RB_SPECIAL_CONST_P(obj)) continue; + bool marked = RVALUE_MARKED(objspace, obj); - if (!RVALUE_MARKED(objspace, obj)) { - **ptr_ptr = Qundef; - } - else { - retained_weak_references_count++; - } + if (marked) { + rgengc_check_relation(objspace, obj); + } + + return marked; +} + +static void +gc_update_weak_references(rb_objspace_t *objspace) +{ + VALUE *obj_ptr; + rb_darray_foreach(objspace->weak_references, i, obj_ptr) { + gc_mark_set_parent(objspace, *obj_ptr); + rb_gc_handle_weak_references(*obj_ptr); + gc_mark_set_parent_invalid(objspace); } - objspace->profile.retained_weak_references_count = retained_weak_references_count; + size_t capa = rb_darray_capa(objspace->weak_references); + size_t size = rb_darray_size(objspace->weak_references); + + objspace->profile.weak_references_count = size; rb_darray_clear(objspace->weak_references); - DURING_GC_COULD_MALLOC_REGION_START(); - { - rb_darray_resize_capa(&objspace->weak_references, retained_weak_references_count); + + /* If the darray has capacity for more than four times the amount used, we + * shrink it down to half of that capacity. */ + if (capa > size * 4) { + rb_darray_resize_capa_without_gc(&objspace->weak_references, size * 2); } - DURING_GC_COULD_MALLOC_REGION_END(); } static void @@ -5485,8 +5632,8 @@ gc_marks_finish(rb_objspace_t *objspace) objspace->flags.during_incremental_marking = FALSE; /* check children of all marked wb-unprotected objects */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - gc_marks_wb_unprotected_objects(objspace, SIZE_POOL_EDEN_HEAP(&size_pools[i])); + for (int i = 0; i < HEAP_COUNT; i++) { + gc_marks_wb_unprotected_objects(objspace, &heaps[i]); } } @@ -5505,7 +5652,7 @@ gc_marks_finish(rb_objspace_t *objspace) { const unsigned long r_mul = objspace->live_ractor_cache_count > 8 ? 8 : objspace->live_ractor_cache_count; // upto 8 - size_t total_slots = heap_eden_total_slots(objspace); + size_t total_slots = objspace_available_slots(objspace); size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */ size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio); size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio); @@ -5515,27 +5662,32 @@ gc_marks_finish(rb_objspace_t *objspace) int full_marking = is_full_marking(objspace); - GC_ASSERT(heap_eden_total_slots(objspace) >= objspace->marked_slots); + GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots); /* Setup freeable slots. */ size_t total_init_slots = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - total_init_slots += gc_params.size_pool_init_slots[i] * r_mul; + for (int i = 0; i < HEAP_COUNT; i++) { + total_init_slots += (gc_params.heap_init_bytes / heaps[i].slot_size) * r_mul; } if (max_free_slots < total_init_slots) { max_free_slots = total_init_slots; } + /* Approximate freeable pages using the average slots-per-pages across all heaps */ if (sweep_slots > max_free_slots) { - heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT; + size_t excess_slots = sweep_slots - max_free_slots; + size_t total_heap_pages = heap_eden_total_pages(objspace); + heap_pages_freeable_pages = total_heap_pages > 0 + ? excess_slots * total_heap_pages / total_slots + : 0; } else { heap_pages_freeable_pages = 0; } - if (objspace->heap_pages.allocatable_slots == 0 && sweep_slots < min_free_slots) { - if (!full_marking) { + if (objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) { + if (!full_marking && sweep_slots < min_free_slots * 7 / 8) { if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) { full_marking = TRUE; } @@ -5544,6 +5696,10 @@ gc_marks_finish(rb_objspace_t *objspace) gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE; } } + + if (full_marking) { + heap_allocatable_bytes_expand(objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size); + } } if (full_marking) { @@ -5562,14 +5718,11 @@ gc_marks_finish(rb_objspace_t *objspace) if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) { gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN; } - if (RGENGC_FORCE_MAJOR_GC) { - gc_needs_major_flags = GPR_FLAG_MAJOR_BY_FORCE; - } gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, " "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, " - "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" slots, next GC: %s)\n", - objspace->marked_slots, objspace->rgengc.old_objects, heap_eden_total_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots, + "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" bytes, next GC: %s)\n", + objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_bytes, gc_needs_major_flags ? "major" : "minor"); } @@ -5586,52 +5739,35 @@ gc_compact_heap_cursors_met_p(rb_heap_t *heap) } -static rb_size_pool_t * -gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, VALUE obj) +static rb_heap_t * +gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE obj) { size_t obj_size = rb_gc_obj_optimal_size(obj); if (obj_size == 0) { return src_pool; } - size_t idx = 0; - if (rb_gc_impl_size_allocatable_p(obj_size)) { - idx = size_pool_idx_for_size(obj_size); - } + GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size)); - return &size_pools[idx]; + size_t idx = heap_idx_for_size(obj_size); + + return &heaps[idx]; } static bool -gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, rb_size_pool_t *size_pool, VALUE src) +gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src) { GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED); GC_ASSERT(gc_is_moveable_obj(objspace, src)); - rb_size_pool_t *dest_pool = gc_compact_destination_pool(objspace, size_pool, src); - rb_heap_t *dheap = SIZE_POOL_EDEN_HEAP(dest_pool); - uint32_t orig_shape = 0; - uint32_t new_shape = 0; - - if (gc_compact_heap_cursors_met_p(dheap)) { - return dheap != heap; - } - - if (RB_TYPE_P(src, T_OBJECT)) { - orig_shape = rb_gc_get_shape(src); - - if (dheap != heap) { - new_shape = rb_gc_rebuild_shape(src, dest_pool - size_pools); - - if (new_shape == 0) { - dheap = heap; - } - } + rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src); + if (gc_compact_heap_cursors_met_p(dest_pool)) { + return dest_pool != heap; } - while (!try_move(objspace, dheap, dheap->free_pages, src)) { + while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) { struct gc_sweep_context ctx = { - .page = dheap->sweeping_page, + .page = dest_pool->sweeping_page, .final_slots = 0, .freed_slots = 0, .empty_slots = 0, @@ -5641,53 +5777,43 @@ gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, rb_size_pool_t *size_p * T_MOVED. Sweeping a page may read objects on this page, so we * need to lock the page. */ lock_page_body(objspace, GET_PAGE_BODY(src)); - gc_sweep_page(objspace, dheap, &ctx); + gc_sweep_page(objspace, dest_pool, &ctx); unlock_page_body(objspace, GET_PAGE_BODY(src)); - if (dheap->sweeping_page->free_slots > 0) { - heap_add_freepage(dheap, dheap->sweeping_page); - } - - dheap->sweeping_page = ccan_list_next(&dheap->pages, dheap->sweeping_page, page_node); - if (gc_compact_heap_cursors_met_p(dheap)) { - return dheap != heap; + if (dest_pool->sweeping_page->free_slots > 0) { + heap_add_freepage(dest_pool, dest_pool->sweeping_page); } - } - if (orig_shape != 0) { - if (new_shape != 0) { - VALUE dest = rb_gc_impl_location(objspace, src); - rb_gc_set_shape(dest, new_shape); + dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node); + if (gc_compact_heap_cursors_met_p(dest_pool)) { + return dest_pool != heap; } - RMOVED(src)->original_shape_id = orig_shape; } return true; } static bool -gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page) +gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page) { short slot_size = page->slot_size; - short slot_bits = slot_size / BASE_SLOT_SIZE; - GC_ASSERT(slot_bits > 0); do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % BASE_SLOT_SIZE == 0); + GC_ASSERT(vp % sizeof(VALUE) == 0); if (bitset & 1) { objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++; if (gc_is_moveable_obj(objspace, vp)) { - if (!gc_compact_move(objspace, heap, size_pool, vp)) { + if (!gc_compact_move(objspace, heap, vp)) { //the cursors met. bubble up return false; } } } p += slot_size; - bitset >>= slot_bits; + bitset >>= 1; } while (bitset); return true; @@ -5695,33 +5821,28 @@ gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t * // Iterate up all the objects in page, moving them to where they want to go static bool -gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, struct heap_page *page) +gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) { GC_ASSERT(page == heap->compact_cursor); bits_t *mark_bits, *pin_bits; bits_t bitset; uintptr_t p = page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); mark_bits = page->mark_bits; pin_bits = page->pinned_bits; - // objects that can be moved are marked and not pinned - bitset = (mark_bits[0] & ~pin_bits[0]); - bitset >>= NUM_IN_PAGE(p); - if (bitset) { - if (!gc_compact_plane(objspace, size_pool, heap, (uintptr_t)p, bitset, page)) - return false; - } - p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; - - for (int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) { + for (int j = 0; j < bitmap_plane_count; j++) { + // objects that can be moved are marked and not pinned bitset = (mark_bits[j] & ~pin_bits[j]); if (bitset) { - if (!gc_compact_plane(objspace, size_pool, heap, (uintptr_t)p, bitset, page)) + if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page)) return false; } - p += BITS_BITLENGTH * BASE_SLOT_SIZE; + p += BITS_BITLENGTH * slot_size; } return true; @@ -5730,9 +5851,8 @@ gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *h static bool gc_compact_all_compacted_p(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; if (heap->total_pages > 0 && !gc_compact_heap_cursors_met_p(heap)) { @@ -5752,9 +5872,8 @@ gc_sweep_compact(rb_objspace_t *objspace) #endif while (!gc_compact_all_compacted_p(objspace)) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; if (gc_compact_heap_cursors_met_p(heap)) { continue; @@ -5762,7 +5881,7 @@ gc_sweep_compact(rb_objspace_t *objspace) struct heap_page *start_page = heap->compact_cursor; - if (!gc_compact_page(objspace, size_pool, heap, start_page)) { + if (!gc_compact_page(objspace, heap, start_page)) { lock_page_body(objspace, start_page->body); continue; @@ -5787,8 +5906,8 @@ gc_marks_rest(rb_objspace_t *objspace) { gc_report(1, objspace, "gc_marks_rest\n"); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - SIZE_POOL_EDEN_HEAP(&size_pools[i])->pooled_pages = NULL; + for (int i = 0; i < HEAP_COUNT; i++) { + (&heaps[i])->pooled_pages = NULL; } if (is_incremental_marking(objspace)) { @@ -5817,7 +5936,7 @@ gc_marks_step(rb_objspace_t *objspace, size_t slots) } static bool -gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap) { GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD); bool marking_finished = true; @@ -5832,7 +5951,7 @@ gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t else { gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n", mark_stack_size(&objspace->mark_stack)); - size_pool->force_incremental_marking_finish_count++; + heap->force_incremental_marking_finish_count++; gc_marks_rest(objspace); } @@ -5866,9 +5985,8 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark) objspace->rgengc.last_major_gc = objspace->profile.count; objspace->marked_slots = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; rgengc_mark_and_rememberset_clear(objspace, heap); heap_move_pooled_pages_to_free_pages(heap); @@ -5887,8 +6005,8 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark) objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */ objspace->profile.minor_gc_count++; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rgengc_rememberset_mark(objspace, SIZE_POOL_EDEN_HEAP(&size_pools[i])); + for (int i = 0; i < HEAP_COUNT; i++) { + rgengc_rememberset_mark(objspace, &heaps[i]); } } @@ -5901,7 +6019,6 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark) static bool gc_marks(rb_objspace_t *objspace, int full_mark) { - gc_prof_mark_timer_start(objspace); gc_marking_enter(objspace); bool marking_finished = false; @@ -5922,7 +6039,6 @@ gc_marks(rb_objspace_t *objspace, int full_mark) #endif gc_marking_exit(objspace); - gc_prof_mark_timer_stop(objspace); return marking_finished; } @@ -6011,7 +6127,7 @@ rgengc_remember(rb_objspace_t *objspace, VALUE obj) #endif static inline void -rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset) +rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset, short slot_size) { if (bitset) { do { @@ -6022,8 +6138,12 @@ rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitse GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj)); gc_mark_children(objspace, obj); + + if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) { + rb_darray_append_without_gc(&objspace->weak_references, obj); + } } - p += BASE_SLOT_SIZE; + p += slot_size; bitset >>= 1; } while (bitset); } @@ -6042,6 +6162,9 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap) ccan_list_for_each(&heap->pages, page, page_node) { if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) { uintptr_t p = page->start; + short slot_size = page->slot_size; + int total_slots = page->total_slots; + int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH); bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT]; bits_t *remembered_bits = page->remembered_bits; bits_t *uncollectible_bits = page->uncollectible_bits; @@ -6051,21 +6174,16 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap) else if (page->flags.has_remembered_objects) has_old++; else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++; #endif - for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) { + for (j=0; j < (size_t)bitmap_plane_count; j++) { bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]); remembered_bits[j] = 0; } page->flags.has_remembered_objects = FALSE; - bitset = bits[0]; - bitset >>= NUM_IN_PAGE(p); - rgengc_rememberset_mark_plane(objspace, p, bitset); - p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; - - for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) { + for (j=0; j < (size_t)bitmap_plane_count; j++) { bitset = bits[j]; - rgengc_rememberset_mark_plane(objspace, p, bitset); - p += BITS_BITLENGTH * BASE_SLOT_SIZE; + rgengc_rememberset_mark_plane(objspace, p, bitset, slot_size); + p += BITS_BITLENGTH * slot_size; } } #if PROFILE_REMEMBERSET_MARK @@ -6112,11 +6230,11 @@ gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace) /* mark `a' and remember (default behavior) */ if (!RVALUE_REMEMBERED(objspace, a)) { - int lev = rb_gc_vm_lock_no_barrier(); + int lev = RB_GC_VM_LOCK_NO_BARRIER(); { rgengc_remember(objspace, a); } - rb_gc_vm_unlock_no_barrier(lev); + RB_GC_VM_UNLOCK_NO_BARRIER(lev); gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b)); } @@ -6130,9 +6248,11 @@ gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent) { gc_mark_set_parent(objspace, parent); rgengc_check_relation(objspace, obj); - if (gc_mark_set(objspace, obj) == FALSE) return; - gc_aging(objspace, obj); - gc_grey(objspace, obj); + if (gc_mark_set(objspace, obj) != FALSE) { + gc_aging(objspace, obj); + gc_grey(objspace, obj); + } + gc_mark_set_parent_invalid(objspace); } NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)); @@ -6164,10 +6284,21 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) { rb_objspace_t *objspace = objspace_ptr; - if (RGENGC_CHECK_MODE) { - if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a); - if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b); - } +#if RGENGC_CHECK_MODE + if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a); + if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b); +#else + RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(a)); + RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(b)); +#endif + + GC_ASSERT(!during_gc); + GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE); + GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED); + GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE); + GC_ASSERT(RB_BUILTIN_TYPE(b) != T_NONE); + GC_ASSERT(RB_BUILTIN_TYPE(b) != T_MOVED); + GC_ASSERT(RB_BUILTIN_TYPE(b) != T_ZOMBIE); retry: if (!is_incremental_marking(objspace)) { @@ -6181,7 +6312,7 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) else { bool retry = false; /* slow path */ - int lev = rb_gc_vm_lock_no_barrier(); + int lev = RB_GC_VM_LOCK_NO_BARRIER(); { if (is_incremental_marking(objspace)) { gc_writebarrier_incremental(a, b, objspace); @@ -6190,7 +6321,7 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) retry = true; } } - rb_gc_vm_unlock_no_barrier(lev); + RB_GC_VM_UNLOCK_NO_BARRIER(lev); if (retry) goto retry; } @@ -6209,7 +6340,7 @@ rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj), RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : ""); - unsigned int lev = rb_gc_vm_lock_no_barrier(); + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); { if (RVALUE_OLD_P(objspace, obj)) { gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj)); @@ -6231,7 +6362,7 @@ rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) RB_DEBUG_COUNTER_INC(obj_wb_unprotect); MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj); } - rb_gc_vm_unlock_no_barrier(lev); + RB_GC_VM_UNLOCK_NO_BARRIER(lev); } } @@ -6241,9 +6372,15 @@ rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) rb_objspace_t *objspace = objspace_ptr; if (RVALUE_WB_UNPROTECTED(objspace, obj)) { - rb_gc_writebarrier_unprotect(dest); + rb_gc_impl_writebarrier_unprotect(objspace, dest); } - rb_gc_copy_finalizer(dest, obj); + rb_gc_impl_copy_finalizer(objspace, dest, obj); +} + +const char * +rb_gc_impl_active_gc_name(void) +{ + return "default"; } void @@ -6253,49 +6390,80 @@ rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj)); - if (is_incremental_marking(objspace)) { - if (RVALUE_BLACK_P(objspace, obj)) { - gc_grey(objspace, obj); - } - } - else { - if (RVALUE_OLD_P(objspace, obj)) { - rgengc_remember(objspace, obj); + if (is_incremental_marking(objspace) || RVALUE_OLD_P(objspace, obj)) { + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + if (is_incremental_marking(objspace)) { + if (RVALUE_BLACK_P(objspace, obj)) { + gc_grey(objspace, obj); + } + } + else if (RVALUE_OLD_P(objspace, obj)) { + rgengc_remember(objspace, obj); + } } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); } } -// TODO: rearchitect this function to work for a generic GC -size_t -rb_gc_impl_obj_flags(void *objspace_ptr, VALUE obj, ID* flags, size_t max) +struct rb_gc_object_metadata_names { + // Must be ID only + ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking, + ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable; +}; + +#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID)) +static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) { rb_objspace_t *objspace = objspace_ptr; size_t n = 0; - static ID ID_marked; - static ID ID_wb_protected, ID_old, ID_marking, ID_uncollectible, ID_pinned; + static struct rb_gc_object_metadata_names names; - if (!ID_marked) { -#define I(s) ID_##s = rb_intern(#s); - I(marked); + if (!names.ID_marked) { +#define I(s) names.ID_##s = rb_intern(#s) I(wb_protected); + I(age); I(old); - I(marking); I(uncollectible); + I(marking); + I(marked); I(pinned); + I(remembered); + I(object_id); + I(shareable); #undef I } - if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0 && n < max) flags[n++] = ID_wb_protected; - if (RVALUE_OLD_P(objspace, obj) && n < max) flags[n++] = ID_old; - if (RVALUE_UNCOLLECTIBLE(objspace, obj) && n < max) flags[n++] = ID_uncollectible; - if (RVALUE_MARKING(objspace, obj) && n < max) flags[n++] = ID_marking; - if (RVALUE_MARKED(objspace, obj) && n < max) flags[n++] = ID_marked; - if (RVALUE_PINNED(objspace, obj) && n < max) flags[n++] = ID_pinned; - return n; +#define SET_ENTRY(na, v) do { \ + GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = names.ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (!RVALUE_WB_UNPROTECTED(objspace, obj)) SET_ENTRY(wb_protected, Qtrue); + SET_ENTRY(age, INT2FIX(RVALUE_AGE_GET(obj))); + if (RVALUE_OLD_P(objspace, obj)) SET_ENTRY(old, Qtrue); + if (RVALUE_UNCOLLECTIBLE(objspace, obj)) SET_ENTRY(uncollectible, Qtrue); + if (RVALUE_MARKING(objspace, obj)) SET_ENTRY(marking, Qtrue); + if (RVALUE_MARKED(objspace, obj)) SET_ENTRY(marked, Qtrue); + if (RVALUE_PINNED(objspace, obj)) SET_ENTRY(pinned, Qtrue); + if (RVALUE_REMEMBERED(objspace, obj)) SET_ENTRY(remembered, Qtrue); + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue); + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; +#undef SET_ENTRY + + return object_metadata_entries; } void * -rb_gc_impl_ractor_cache_alloc(void *objspace_ptr) +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) { rb_objspace_t *objspace = objspace_ptr; @@ -6310,18 +6478,17 @@ rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache) rb_objspace_t *objspace = objspace_ptr; objspace->live_ractor_cache_count--; - gc_ractor_newobj_cache_clear(cache, NULL); free(cache); } static void -heap_ready_to_gc(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) +heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap) { if (!heap->free_pages) { - if (!heap_page_allocate_and_initialize(objspace, size_pool, heap)) { - objspace->heap_pages.allocatable_slots = 1; - heap_page_allocate_and_initialize(objspace, size_pool, heap); + if (!heap_page_allocate_and_initialize(objspace, heap)) { + objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE; + heap_page_allocate_and_initialize(objspace, heap); } } } @@ -6329,10 +6496,10 @@ heap_ready_to_gc(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t * static int ready_to_gc(rb_objspace_t *objspace) { - if (dont_gc_val() || during_gc || ruby_disable_gc) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - heap_ready_to_gc(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool)); + if (dont_gc_val() || during_gc) { + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + heap_ready_to_gc(objspace, heap); } return FALSE; } @@ -6346,11 +6513,14 @@ gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark) { gc_prof_set_malloc_info(objspace); { - size_t inc = RUBY_ATOMIC_SIZE_EXCHANGE(malloc_increase, 0); + int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters); size_t old_limit = malloc_limit; - if (inc > malloc_limit) { - malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor); + /* A net-negative `inc` (more freed than malloc'd since last GC) is + * treated the same as "allocated less than malloc_limit". + * This matches what we were doing pre-monotonic counters, but is it right? */ + if (inc > 0 && (size_t)inc > malloc_limit) { + malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor); if (malloc_limit > gc_params.malloc_limit_max) { malloc_limit = gc_params.malloc_limit_max; } @@ -6377,7 +6547,11 @@ gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark) /* reset oldmalloc info */ #if RGENGC_ESTIMATE_OLDMALLOC if (!full_mark) { - if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) { + /* Don't snapshot on minor GC: oldmalloc_increase is meant to + * accumulate across minor GCs and only reset at major GC. */ + int64_t oldmalloc_increase = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.oldcounters); + if (oldmalloc_increase > 0 && + (uint64_t)oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) { gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC; objspace->rgengc.oldmalloc_increase_limit = (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor); @@ -6387,17 +6561,14 @@ gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark) } } - if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRIuSIZE"\t%"PRIuSIZE"\t%"PRIdSIZE"\n", + if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRId64"\t%"PRIuSIZE"\t%"PRIdSIZE"\n", rb_gc_count(), gc_needs_major_flags, - objspace->rgengc.oldmalloc_increase, + oldmalloc_increase, objspace->rgengc.oldmalloc_increase_limit, gc_params.oldmalloc_limit_max); } else { - /* major GC */ - objspace->rgengc.oldmalloc_increase = 0; - if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) { objspace->rgengc.oldmalloc_increase_limit = (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1)); @@ -6414,7 +6585,7 @@ garbage_collect(rb_objspace_t *objspace, unsigned int reason) { int ret; - int lev = rb_gc_vm_lock(); + int lev = RB_GC_VM_LOCK(); { #if GC_PROFILE_MORE_DETAIL objspace->profile.prepare_time = getrusage_time(); @@ -6428,7 +6599,7 @@ garbage_collect(rb_objspace_t *objspace, unsigned int reason) ret = gc_start(objspace, reason); } - rb_gc_vm_unlock(lev); + RB_GC_VM_UNLOCK(lev); return ret; } @@ -6438,19 +6609,21 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) { unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK); - /* reason may be clobbered, later, so keep set immediate_sweep here */ - objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); - if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */ if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */ - GC_ASSERT(gc_mode(objspace) == gc_mode_none); + rb_gc_initialize_vm_context(&objspace->vm_context); + + GC_ASSERT(gc_mode(objspace) == gc_mode_none, "gc_mode is %s\n", gc_mode_name(gc_mode(objspace))); GC_ASSERT(!is_lazy_sweeping(objspace)); GC_ASSERT(!is_incremental_marking(objspace)); unsigned int lock_lev; gc_enter(objspace, gc_enter_event_start, &lock_lev); + /* reason may be clobbered, later, so keep set immediate_sweep here */ + objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); + #if RGENGC_CHECK_MODE >= 2 gc_verify_internal_consistency(objspace); #endif @@ -6469,10 +6642,6 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) reason |= gc_needs_major_flags; do_full_mark = TRUE; } - else if (RGENGC_FORCE_MAJOR_GC) { - reason = GPR_FLAG_MAJOR_BY_FORCE; - do_full_mark = TRUE; - } /* if major gc has been disabled, never do a full mark */ if (!gc_config_full_mark_val) { @@ -6514,7 +6683,6 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) reason, do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep); -#if USE_DEBUG_COUNTER RB_DEBUG_COUNTER_INC(gc_count); if (reason & GPR_FLAG_MAJOR_MASK) { @@ -6533,14 +6701,13 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI); (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS); } -#endif objspace->profile.count++; objspace->profile.latest_gc_info = reason; objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace); objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted); + objspace->profile.heap_total_slots_at_gc_start = objspace_available_slots(objspace); objspace->profile.weak_references_count = 0; - objspace->profile.retained_weak_references_count = 0; gc_prof_setup_new_record(objspace, reason); gc_reset_malloc_info(objspace, do_full_mark); @@ -6700,7 +6867,7 @@ gc_clock_start(struct timespec *ts) } } -static uint64_t +static unsigned long long gc_clock_end(struct timespec *ts) { struct timespec end_time; @@ -6708,7 +6875,7 @@ gc_clock_end(struct timespec *ts) if ((ts->tv_sec > 0 || ts->tv_nsec > 0) && current_process_time(&end_time) && end_time.tv_sec >= ts->tv_sec) { - return (uint64_t)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) + + return (unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) + (end_time.tv_nsec - ts->tv_nsec); } @@ -6718,12 +6885,10 @@ gc_clock_end(struct timespec *ts) static inline void gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev) { - *lock_lev = rb_gc_vm_lock(); + *lock_lev = RB_GC_VM_LOCK(); switch (event) { case gc_enter_event_rest: - if (!is_marking(objspace)) break; - // fall through case gc_enter_event_start: case gc_enter_event_continue: // stop other ractors @@ -6757,7 +6922,7 @@ gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_l gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace)); during_gc = FALSE; - rb_gc_vm_unlock(*lock_lev); + RB_GC_VM_UNLOCK(*lock_lev); } #ifndef MEASURE_GC @@ -6769,9 +6934,13 @@ gc_marking_enter(rb_objspace_t *objspace) { GC_ASSERT(during_gc != 0); + gc_prof_mark_timer_start(objspace); + if (MEASURE_GC) { gc_clock_start(&objspace->profile.marking_start_time); } + + rb_gc_initialize_vm_context(&objspace->vm_context); } static void @@ -6782,6 +6951,8 @@ gc_marking_exit(rb_objspace_t *objspace) if (MEASURE_GC) { objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time); } + + gc_prof_mark_timer_stop(objspace); } static void @@ -6816,22 +6987,23 @@ int ruby_thread_has_gvl_p(void); static int garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason) { - if (dont_gc_val()) return TRUE; - if (ruby_thread_has_gvl_p()) { - return garbage_collect(objspace, reason); + if (dont_gc_val()) { + return TRUE; + } + else if (!ruby_native_thread_p()) { + return TRUE; + } + else if (!ruby_thread_has_gvl_p()) { + void *ret; + struct objspace_and_reason oar; + oar.objspace = objspace; + oar.reason = reason; + ret = rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar); + + return !!ret; } else { - if (ruby_native_thread_p()) { - struct objspace_and_reason oar; - oar.objspace = objspace; - oar.reason = reason; - return (int)(VALUE)rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar); - } - else { - /* no ruby thread */ - fprintf(stderr, "[FATAL] failed to allocate memory\n"); - exit(EXIT_FAILURE); - } + return garbage_collect(objspace, reason); } } @@ -6847,12 +7019,8 @@ gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data) case T_NONE: case T_ZOMBIE: break; - case T_STRING: - // precompute the string coderange. This both save time for when it will be - // eventually needed, and avoid mutating heap pages after a potential fork. - rb_enc_str_coderange(v); - // fall through default: + rb_gc_prepare_heap_process_object(v); if (!RVALUE_OLD_P(objspace, v) && !RVALUE_WB_UNPROTECTED(objspace, v)) { RVALUE_AGE_SET_CANDIDATE(objspace, v); } @@ -6898,8 +7066,8 @@ rb_gc_impl_prepare_heap(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - size_t orig_total_slots = heap_eden_total_slots(objspace); - size_t orig_allocatable_slots = objspace->heap_pages.allocatable_slots; + size_t orig_total_slots = objspace_available_slots(objspace); + size_t orig_allocatable_bytes = objspace->heap_pages.allocatable_bytes; rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr); @@ -6909,14 +7077,16 @@ rb_gc_impl_prepare_heap(void *objspace_ptr) rb_gc_impl_start(objspace, true, true, true, true); gc_params.heap_free_slots_max_ratio = orig_max_free_slots; - objspace->heap_pages.allocatable_slots = 0; + objspace->heap_pages.allocatable_bytes = 0; + heap_pages_freeable_pages = objspace->empty_pages_count; heap_pages_free_unused_pages(objspace_ptr); + GC_ASSERT(heap_pages_freeable_pages == 0); GC_ASSERT(objspace->empty_pages_count == 0); - objspace->heap_pages.allocatable_slots = orig_allocatable_slots; + objspace->heap_pages.allocatable_bytes = orig_allocatable_bytes; - size_t total_slots = heap_eden_total_slots(objspace); + size_t total_slots = objspace_available_slots(objspace); if (orig_total_slots > total_slots) { - objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots; + objspace->heap_pages.allocatable_bytes += (orig_total_slots - total_slots) * heaps[0].slot_size; } #if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER) @@ -6935,12 +7105,6 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj) case T_ZOMBIE: return FALSE; case T_SYMBOL: - // TODO: restore original behavior - // if (RSYMBOL(obj)->id & ~ID_SCOPE_MASK) { - // return FALSE; - // } - return false; - /* fall through */ case T_STRING: case T_OBJECT: case T_FLOAT: @@ -6959,7 +7123,7 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj) case T_RATIONAL: case T_NODE: case T_CLASS: - if (FL_TEST(obj, FL_FINALIZE)) { + if (FL_TEST_RAW(obj, FL_FINALIZE)) { /* The finalizer table is a numtable. It looks up objects by address. * We can't mark the keys in the finalizer table because that would * prevent the objects from being collected. This check prevents @@ -6985,8 +7149,11 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj) void rb_mv_generic_ivar(VALUE src, VALUE dst); static VALUE -gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, size_t slot_size) +gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_page, struct heap_page *dest_page) { + size_t src_slot_size = src_page->slot_size; + size_t slot_size = dest_page->slot_size; + int marked; int wb_unprotected; int uncollectible; @@ -7012,39 +7179,13 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, si CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src); CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src); - if (FL_TEST(src, FL_EXIVAR)) { - /* Resizing the st table could cause a malloc */ - DURING_GC_COULD_MALLOC_REGION_START(); - { - rb_mv_generic_ivar(src, dest); - } - DURING_GC_COULD_MALLOC_REGION_END(); - } - - if (FL_TEST(src, FL_SEEN_OBJ_ID)) { - /* If the source object's object_id has been seen, we need to update - * the object to object id mapping. */ - st_data_t srcid = (st_data_t)src, id; - - gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest); - /* Resizing the st table could cause a malloc */ - DURING_GC_COULD_MALLOC_REGION_START(); - { - if (!st_delete(objspace->obj_to_id_tbl, &srcid, &id)) { - rb_bug("gc_move: object ID seen, but not in mapping table: %s", rb_obj_info((VALUE)src)); - } - - st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id); - } - DURING_GC_COULD_MALLOC_REGION_END(); - } - else { - GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)src, NULL)); - } - /* Move the object */ memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size)); + if (src_slot_size != slot_size && RB_TYPE_P(src, T_OBJECT)) { + rb_gc_obj_changed_pool(dest, dest_page->heap - heaps); + } + if (RVALUE_OVERHEAD > 0) { void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD); void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD); @@ -7053,7 +7194,7 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, si } memset((void *)src, 0, src_slot_size); - RVALUE_AGE_RESET(src); + RVALUE_AGE_SET_BITMAP(src, 0); /* Set bits for object in new location */ if (remembered) { @@ -7091,8 +7232,8 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, si RMOVED(src)->destination = dest; GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE); - GET_HEAP_PAGE(src)->size_pool->total_freed_objects++; - GET_HEAP_PAGE(dest)->size_pool->total_allocated_objects++; + GET_HEAP_PAGE(src)->heap->total_freed_objects++; + GET_HEAP_PAGE(dest)->heap->total_allocated_objects++; return src; } @@ -7125,16 +7266,16 @@ compare_free_slots(const void *left, const void *right, void *dummy) static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func) { - for (int j = 0; j < SIZE_POOL_COUNT; j++) { - rb_size_pool_t *size_pool = &size_pools[j]; + for (int j = 0; j < HEAP_COUNT; j++) { + rb_heap_t *heap = &heaps[j]; - size_t total_pages = SIZE_POOL_EDEN_HEAP(size_pool)->total_pages; + size_t total_pages = heap->total_pages; size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError); struct heap_page *page = 0, **page_list = malloc(size); size_t i = 0; - SIZE_POOL_EDEN_HEAP(size_pool)->free_pages = NULL; - ccan_list_for_each(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node) { + heap->free_pages = NULL; + ccan_list_for_each(&heap->pages, page, page_node) { page_list[i++] = page; GC_ASSERT(page); } @@ -7146,12 +7287,12 @@ gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func co ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL); /* Reset the eden heap */ - ccan_list_head_init(&SIZE_POOL_EDEN_HEAP(size_pool)->pages); + ccan_list_head_init(&heap->pages); for (i = 0; i < total_pages; i++) { - ccan_list_add(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, &page_list[i]->page_node); + ccan_list_add(&heap->pages, &page_list[i]->page_node); if (page_list[i]->free_slots != 0) { - heap_add_freepage(SIZE_POOL_EDEN_HEAP(size_pool), page_list[i]); + heap_add_freepage(heap, page_list[i]); } } @@ -7160,6 +7301,12 @@ gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func co } #endif +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + /* no-op */ +} + bool rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) { @@ -7170,62 +7317,76 @@ static int gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t *objspace, struct heap_page *page) { VALUE v = (VALUE)vstart; - asan_unlock_freelist(page); - asan_lock_freelist(page); + page->flags.has_uncollectible_wb_unprotected_objects = FALSE; page->flags.has_remembered_objects = FALSE; /* For each object on the page */ for (; v != (VALUE)vend; v += stride) { - void *poisoned = asan_unpoison_object_temporary(v); - - switch (BUILTIN_TYPE(v)) { - case T_NONE: - case T_MOVED: - case T_ZOMBIE: - break; - default: - if (RVALUE_WB_UNPROTECTED(objspace, v)) { - page->flags.has_uncollectible_wb_unprotected_objects = TRUE; - } - if (RVALUE_REMEMBERED(objspace, v)) { - page->flags.has_remembered_objects = TRUE; - } - if (page->flags.before_sweep) { - if (RVALUE_MARKED(objspace, v)) { + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_MOVED: + case T_ZOMBIE: + break; + default: + if (RVALUE_WB_UNPROTECTED(objspace, v)) { + page->flags.has_uncollectible_wb_unprotected_objects = TRUE; + } + if (RVALUE_REMEMBERED(objspace, v)) { + page->flags.has_remembered_objects = TRUE; + } + if (page->flags.before_sweep) { + if (RVALUE_MARKED(objspace, v)) { + rb_gc_update_object_references(objspace, v); + } + } + else { rb_gc_update_object_references(objspace, v); } } - else { - rb_gc_update_object_references(objspace, v); - } - } - - if (poisoned) { - asan_poison_object(v); } } return 0; } +static int +gc_update_references_weak_table_i(VALUE obj, void *data) +{ + int ret; + asan_unpoisoning_object(obj) { + ret = BUILTIN_TYPE(obj) == T_MOVED ? ST_REPLACE : ST_CONTINUE; + } + return ret; +} + +static int +gc_update_references_weak_table_replace_i(VALUE *obj, void *data) +{ + *obj = rb_gc_location(*obj); + + return ST_CONTINUE; +} + static void gc_update_references(rb_objspace_t *objspace) { objspace->flags.during_reference_updating = true; + rb_gc_before_updating_jit_code(); + struct heap_page *page = NULL; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { + for (int i = 0; i < HEAP_COUNT; i++) { bool should_set_mark_bits = TRUE; - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + rb_heap_t *heap = &heaps[i]; ccan_list_for_each(&heap->pages, page, page_node) { uintptr_t start = (uintptr_t)page->start; - uintptr_t end = start + (page->total_slots * size_pool->slot_size); + uintptr_t end = start + (page->total_slots * heap->slot_size); - gc_ref_update((void *)start, (void *)end, size_pool->slot_size, objspace, page); + gc_ref_update((void *)start, (void *)end, heap->slot_size, objspace, page); if (page == heap->sweeping_page) { should_set_mark_bits = FALSE; } @@ -7234,12 +7395,23 @@ gc_update_references(rb_objspace_t *objspace) } } } - gc_ref_update_table_values_only(objspace->obj_to_id_tbl); - gc_update_table_refs(objspace->id_to_obj_tbl); + gc_update_table_refs(finalizer_table); rb_gc_update_vm_references((void *)objspace); + for (int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) { + rb_gc_vm_weak_table_foreach( + gc_update_references_weak_table_i, + gc_update_references_weak_table_replace_i, + NULL, + false, + table + ); + } + + rb_gc_after_updating_jit_code(); + objspace->flags.during_reference_updating = false; } @@ -7274,22 +7446,17 @@ heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data) /* Moved object still on the heap, something may have a reference. */ } else { - void *poisoned = asan_unpoison_object_temporary(v); - - switch (BUILTIN_TYPE(v)) { - case T_NONE: - case T_ZOMBIE: - break; - default: - if (!rb_gc_impl_garbage_object_p(objspace, v)) { - rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v); + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_ZOMBIE: + break; + default: + if (!rb_gc_impl_garbage_object_p(objspace, v)) { + rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v); + } } } - - if (poisoned) { - GC_ASSERT(BUILTIN_TYPE(v) == T_NONE); - asan_poison_object(v); - } } } @@ -7297,38 +7464,6 @@ heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data) } #endif -struct desired_compaction_pages_i_data { - rb_objspace_t *objspace; - size_t required_slots[SIZE_POOL_COUNT]; -}; - -static int -desired_compaction_pages_i(struct heap_page *page, void *data) -{ - struct desired_compaction_pages_i_data *tdata = data; - rb_objspace_t *objspace = tdata->objspace; - VALUE vstart = (VALUE)page->start; - VALUE vend = vstart + (VALUE)(page->total_slots * page->size_pool->slot_size); - - - for (VALUE v = vstart; v != vend; v += page->size_pool->slot_size) { - /* skip T_NONEs; they won't be moved */ - void *poisoned = asan_unpoison_object_temporary(v); - if (BUILTIN_TYPE(v) == T_NONE) { - if (poisoned) { - asan_poison_object(v); - } - continue; - } - - rb_size_pool_t *dest_pool = gc_compact_destination_pool(objspace, page->size_pool, v); - size_t dest_pool_idx = dest_pool - size_pools; - tdata->required_slots[dest_pool_idx]++; - } - - return 0; -} - bool rb_gc_impl_during_gc_p(void *objspace_ptr) { @@ -7405,7 +7540,7 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned #endif static VALUE sym_newobj, sym_malloc, sym_method, sym_capi; static VALUE sym_none, sym_marking, sym_sweeping; - static VALUE sym_weak_references_count, sym_retained_weak_references_count; + static VALUE sym_weak_references_count; VALUE hash = Qnil, key = Qnil; VALUE major_by, need_major_by; unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info; @@ -7417,7 +7552,7 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned hash = hash_or_key; } else { - rb_raise(rb_eTypeError, "non-hash or symbol given"); + rb_bug("gc_info_decode: non-hash or symbol given"); } if (NIL_P(sym_major_by)) { @@ -7447,7 +7582,6 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned S(sweeping); S(weak_references_count); - S(retained_weak_references_count); #undef S } @@ -7500,11 +7634,11 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned } SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count)); - SET(retained_weak_references_count, LONG2FIX(objspace->profile.retained_weak_references_count)); #undef SET - if (!NIL_P(key)) {/* matched key should return above */ - rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; } return hash; @@ -7526,7 +7660,7 @@ enum gc_stat_sym { gc_stat_sym_sweeping_time, gc_stat_sym_heap_allocated_pages, gc_stat_sym_heap_empty_pages, - gc_stat_sym_heap_allocatable_slots, + gc_stat_sym_heap_allocatable_bytes, gc_stat_sym_heap_available_slots, gc_stat_sym_heap_live_slots, gc_stat_sym_heap_free_slots, @@ -7537,6 +7671,8 @@ enum gc_stat_sym { gc_stat_sym_total_freed_pages, gc_stat_sym_total_allocated_objects, gc_stat_sym_total_freed_objects, + gc_stat_sym_total_malloc_bytes, + gc_stat_sym_total_free_bytes, gc_stat_sym_malloc_increase_bytes, gc_stat_sym_malloc_increase_bytes_limit, gc_stat_sym_minor_gc_count, @@ -7552,7 +7688,6 @@ enum gc_stat_sym { gc_stat_sym_oldmalloc_increase_bytes, gc_stat_sym_oldmalloc_increase_bytes_limit, #endif - gc_stat_sym_weak_references_count, #if RGENGC_PROFILE gc_stat_sym_total_generated_normal_object_count, gc_stat_sym_total_generated_shady_object_count, @@ -7577,7 +7712,7 @@ setup_gc_stat_symbols(void) S(sweeping_time), S(heap_allocated_pages); S(heap_empty_pages); - S(heap_allocatable_slots); + S(heap_allocatable_bytes); S(heap_available_slots); S(heap_live_slots); S(heap_free_slots); @@ -7588,6 +7723,8 @@ setup_gc_stat_symbols(void) S(total_freed_pages); S(total_allocated_objects); S(total_freed_objects); + S(total_malloc_bytes); + S(total_free_bytes); S(malloc_increase_bytes); S(malloc_increase_bytes_limit); S(minor_gc_count); @@ -7603,7 +7740,6 @@ setup_gc_stat_symbols(void) S(oldmalloc_increase_bytes); S(oldmalloc_increase_bytes_limit); #endif - S(weak_references_count); #if RGENGC_PROFILE S(total_generated_normal_object_count); S(total_generated_shady_object_count); @@ -7622,7 +7758,9 @@ ns_to_ms(uint64_t ns) return ns / (1000 * 1000); } -size_t +static void malloc_increase_local_flush(rb_objspace_t *objspace); + +VALUE rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) { rb_objspace_t *objspace = objspace_ptr; @@ -7630,6 +7768,9 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) setup_gc_stat_symbols(); + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + malloc_increase_local_flush(objspace); + if (RB_TYPE_P(hash_or_sym, T_HASH)) { hash = hash_or_sym; } @@ -7637,35 +7778,40 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) key = hash_or_sym; } else { - rb_raise(rb_eTypeError, "non-hash or symbol given"); + rb_bug("non-hash or symbol given"); } #define SET(name, attr) \ if (key == gc_stat_symbols[gc_stat_sym_##name]) \ - return attr; \ + return SIZET2NUM(attr); \ else if (hash != Qnil) \ rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr)); +#define SET64(name, attr) \ + if (key == gc_stat_symbols[gc_stat_sym_##name]) \ + return ULL2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr)); SET(count, objspace->profile.count); SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns)); SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns)); - /* implementation dependent counters */ + { + uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc); + uint64_t total_free = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.free); + SET64(total_malloc_bytes, total_malloc); + SET64(total_free_bytes, total_free); + } + + /* implementation dependent counters (small / fixnum-safe) */ SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted)); SET(heap_empty_pages, objspace->empty_pages_count) - SET(heap_allocatable_slots, objspace->heap_pages.allocatable_slots); - SET(heap_available_slots, objspace_available_slots(objspace)); - SET(heap_live_slots, objspace_live_slots(objspace)); - SET(heap_free_slots, objspace_free_slots(objspace)); - SET(heap_final_slots, total_final_slots_count(objspace)); - SET(heap_marked_slots, objspace->marked_slots); + SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes); SET(heap_eden_pages, heap_eden_total_pages(objspace)); SET(total_allocated_pages, objspace->heap_pages.allocated_pages); SET(total_freed_pages, objspace->heap_pages.freed_pages); - SET(total_allocated_objects, total_allocated_objects(objspace)); - SET(total_freed_objects, total_freed_objects(objspace)); - SET(malloc_increase_bytes, malloc_increase); + SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)); SET(malloc_increase_bytes_limit, malloc_limit); SET(minor_gc_count, objspace->profile.minor_gc_count); SET(major_gc_count, objspace->profile.major_gc_count); @@ -7677,10 +7823,19 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) SET(old_objects, objspace->rgengc.old_objects); SET(old_objects_limit, objspace->rgengc.old_objects_limit); #if RGENGC_ESTIMATE_OLDMALLOC - SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase); + SET(oldmalloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.oldcounters)); SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit); #endif + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + SET(total_allocated_objects, total_allocated_objects(objspace)); + SET(total_freed_objects, total_freed_objects(objspace)); + SET(heap_available_slots, objspace_available_slots(objspace)); + SET(heap_live_slots, objspace_live_slots(objspace)); + SET(heap_free_slots, objspace_free_slots(objspace)); + SET(heap_final_slots, total_final_slots_count(objspace)); + SET(heap_marked_slots, objspace->marked_slots); + #if RGENGC_PROFILE SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count); SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count); @@ -7690,9 +7845,11 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count); #endif /* RGENGC_PROFILE */ #undef SET +#undef SET64 - if (!NIL_P(key)) { /* matched key should return above */ - rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; } #if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2 @@ -7706,16 +7863,20 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) } #endif - return 0; + return hash; } enum gc_stat_heap_sym { gc_stat_heap_sym_slot_size, + gc_stat_heap_sym_heap_live_slots, + gc_stat_heap_sym_heap_free_slots, + gc_stat_heap_sym_heap_final_slots, gc_stat_heap_sym_heap_eden_pages, gc_stat_heap_sym_heap_eden_slots, gc_stat_heap_sym_total_allocated_pages, gc_stat_heap_sym_force_major_gc_count, gc_stat_heap_sym_force_incremental_marking_finish_count, + gc_stat_heap_sym_heap_allocatable_slots, gc_stat_heap_sym_total_allocated_objects, gc_stat_heap_sym_total_freed_objects, gc_stat_heap_sym_last @@ -7729,8 +7890,12 @@ setup_gc_stat_heap_symbols(void) if (gc_stat_heap_symbols[0] == 0) { #define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s)) S(slot_size); + S(heap_live_slots); + S(heap_free_slots); + S(heap_final_slots); S(heap_eden_pages); S(heap_eden_slots); + S(heap_allocatable_slots); S(total_allocated_pages); S(force_major_gc_count); S(force_incremental_marking_finish_count); @@ -7740,76 +7905,83 @@ setup_gc_stat_heap_symbols(void) } } -static size_t -stat_one_heap(rb_size_pool_t *size_pool, VALUE hash, VALUE key) +static VALUE +stat_one_heap(rb_objspace_t *objspace, rb_heap_t *heap, VALUE hash, VALUE key) { #define SET(name, attr) \ if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \ - return attr; \ + return SIZET2NUM(attr); \ else if (hash != Qnil) \ rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr)); - SET(slot_size, size_pool->slot_size); - SET(heap_eden_pages, SIZE_POOL_EDEN_HEAP(size_pool)->total_pages); - SET(heap_eden_slots, SIZE_POOL_EDEN_HEAP(size_pool)->total_slots); - SET(total_allocated_pages, size_pool->total_allocated_pages); - SET(force_major_gc_count, size_pool->force_major_gc_count); - SET(force_incremental_marking_finish_count, size_pool->force_incremental_marking_finish_count); - SET(total_allocated_objects, size_pool->total_allocated_objects); - SET(total_freed_objects, size_pool->total_freed_objects); + SET(slot_size, heap->slot_size); + SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count); + SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects)); + SET(heap_final_slots, heap->final_slots_count); + SET(heap_eden_pages, heap->total_pages); + SET(heap_eden_slots, heap->total_slots); + SET(heap_allocatable_slots, objspace->heap_pages.allocatable_bytes / heap->slot_size); + SET(total_allocated_pages, heap->total_allocated_pages); + SET(force_major_gc_count, heap->force_major_gc_count); + SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count); + SET(total_allocated_objects, heap->total_allocated_objects); + SET(total_freed_objects, heap->total_freed_objects); #undef SET - if (!NIL_P(key)) { /* matched key should return above */ - rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; } - return 0; + return hash; } -size_t +VALUE rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) { rb_objspace_t *objspace = objspace_ptr; + ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache()); + setup_gc_stat_heap_symbols(); if (NIL_P(heap_name)) { if (!RB_TYPE_P(hash_or_sym, T_HASH)) { - rb_raise(rb_eTypeError, "non-hash given"); + rb_bug("non-hash given"); } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { + for (int i = 0; i < HEAP_COUNT; i++) { VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i)); if (NIL_P(hash)) { hash = rb_hash_new(); rb_hash_aset(hash_or_sym, INT2FIX(i), hash); } - stat_one_heap(&size_pools[i], hash, Qnil); + stat_one_heap(objspace, &heaps[i], hash, Qnil); } } else if (FIXNUM_P(heap_name)) { - int size_pool_idx = FIX2INT(heap_name); + int heap_idx = FIX2INT(heap_name); - if (size_pool_idx < 0 || size_pool_idx >= SIZE_POOL_COUNT) { + if (heap_idx < 0 || heap_idx >= HEAP_COUNT) { rb_raise(rb_eArgError, "size pool index out of range"); } if (SYMBOL_P(hash_or_sym)) { - return stat_one_heap(&size_pools[size_pool_idx], Qnil, hash_or_sym); + return stat_one_heap(objspace, &heaps[heap_idx], Qnil, hash_or_sym); } else if (RB_TYPE_P(hash_or_sym, T_HASH)) { - return stat_one_heap(&size_pools[size_pool_idx], hash_or_sym, Qnil); + return stat_one_heap(objspace, &heaps[heap_idx], hash_or_sym, Qnil); } else { - rb_raise(rb_eTypeError, "non-hash or symbol given"); + rb_bug("non-hash or symbol given"); } } else { - rb_raise(rb_eTypeError, "heap_name must be nil or an Integer"); + rb_bug("heap_name must be nil or an Integer"); } - return 0; + return hash_or_sym; } /* I could include internal.h for this, but doing so undefines some Array macros @@ -7834,7 +8006,7 @@ rb_gc_impl_config_get(void *objspace_ptr) } static int -gc_config_set_key(st_data_t key, st_data_t value, st_data_t data) +gc_config_set_key(VALUE key, VALUE value, VALUE data) { rb_objspace_t *objspace = (rb_objspace_t *)data; if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) { @@ -7844,17 +8016,16 @@ gc_config_set_key(st_data_t key, st_data_t value, st_data_t data) return ST_CONTINUE; } -VALUE +void rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) { rb_objspace_t *objspace = objspace_ptr; - if(!RB_TYPE_P(hash, T_HASH)) { + if (!RB_TYPE_P(hash, T_HASH)) { rb_raise(rb_eArgError, "expected keyword arguments"); } - rb_hash_stlike_foreach(hash, gc_config_set_key, (st_data_t)objspace); - return rb_gc_impl_config_get(objspace_ptr); + rb_hash_foreach(hash, gc_config_set_key, (st_data_t)objspace); } VALUE @@ -7982,8 +8153,8 @@ get_envparam_double(const char *name, double *default_value, double lower_bound, * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1) * - Allocate slots by this factor. * - (next slots number) = (current slots number) * (this factor) - * * RUBY_GC_HEAP_GROWTH_MAX_SLOTS (new from 2.1) - * - Allocation rate is limited to this number of slots. + * * RUBY_GC_HEAP_GROWTH_MAX_BYTES (was RUBY_GC_HEAP_GROWTH_MAX_SLOTS) + * - Allocation rate is limited to this number of bytes. * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4) * - Allocate additional pages when the number of free slots is * lower than the value (total_slots * (this ratio)). @@ -8002,7 +8173,7 @@ get_envparam_double(const char *name, double *default_value, double lower_bound, * * * obsolete * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1) - * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) + * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) -> RUBY_GC_HEAP_INIT_BYTES * * * RUBY_GC_MALLOC_LIMIT * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1) @@ -8022,15 +8193,10 @@ rb_gc_impl_set_params(void *objspace_ptr) /* ok */ } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)]; - snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i); - - get_envparam_size(env_key, &gc_params.size_pool_init_slots[i], 0); - } + get_envparam_size("RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes, 0); get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE); - get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0); + get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0); get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio, 0.0, 1.0, FALSE); get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio, @@ -8062,10 +8228,11 @@ static inline size_t objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint) { #ifdef HAVE_MALLOC_USABLE_SIZE - return malloc_usable_size(ptr); -#else - return hint; + if (!hint) { + hint = malloc_usable_size(ptr); + } #endif + return hint; } enum memop_type { @@ -8103,8 +8270,53 @@ objspace_malloc_gc_stress(rb_objspace_t *objspace) } } +static void +malloc_increase_commit(rb_objspace_t *objspace, size_t new_size, size_t old_size) +{ + if (new_size > old_size) { + size_t delta = new_size - old_size; + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_add(&objspace->malloc_counters.counters.malloc, delta); +#if RGENGC_ESTIMATE_OLDMALLOC + gc_counter_add(&objspace->malloc_counters.oldcounters.malloc, delta); +#endif + MALLOC_COUNTERS_UNLOCK(objspace); + } + else if (old_size > new_size) { + size_t delta = old_size - new_size; + MALLOC_COUNTERS_LOCK(objspace); + gc_counter_add(&objspace->malloc_counters.counters.free, delta); +#if RGENGC_ESTIMATE_OLDMALLOC + gc_counter_add(&objspace->malloc_counters.oldcounters.free, delta); +#endif + MALLOC_COUNTERS_UNLOCK(objspace); + } +} + +#if USE_MALLOC_INCREASE_LOCAL +static void +malloc_increase_local_flush(rb_objspace_t *objspace) +{ + int delta = malloc_increase_local; + if (delta == 0) return; + + malloc_increase_local = 0; + if (delta > 0) { + malloc_increase_commit(objspace, (size_t)delta, 0); + } + else { + malloc_increase_commit(objspace, 0, (size_t)(-delta)); + } +} +#else +static void +malloc_increase_local_flush(rb_objspace_t *objspace) +{ +} +#endif + static inline bool -objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type) +objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed) { if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n", mem, @@ -8116,22 +8328,27 @@ objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_s } static bool -objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type) +objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed) { - if (new_size > old_size) { - RUBY_ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size); -#if RGENGC_ESTIMATE_OLDMALLOC - RUBY_ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size); -#endif +#if USE_MALLOC_INCREASE_LOCAL + if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD && + old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) { + malloc_increase_local += (int)new_size - (int)old_size; + + if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD || + malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) { + malloc_increase_local_flush(objspace); + } } else { - atomic_sub_nounderflow(&malloc_increase, old_size - new_size); -#if RGENGC_ESTIMATE_OLDMALLOC - atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size); -#endif + malloc_increase_local_flush(objspace); + malloc_increase_commit(objspace, new_size, old_size); } +#else + malloc_increase_commit(objspace, new_size, old_size); +#endif - if (type == MEMOP_TYPE_MALLOC) { + if (type == MEMOP_TYPE_MALLOC && gc_allowed) { retry: if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) { @@ -8148,9 +8365,9 @@ objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_siz } else { size_t dec_size = old_size - new_size; - size_t allocated_size = objspace->malloc_params.allocated_size; #if MALLOC_ALLOCATED_SIZE_CHECK + size_t allocated_size = objspace->malloc_params.allocated_size; if (allocated_size < dec_size) { rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size."); } @@ -8213,10 +8430,10 @@ malloc_during_gc_p(rb_objspace_t *objspace) } static inline void * -objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size) +objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size, bool gc_allowed) { size = objspace_malloc_size(objspace, mem, size); - objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {} + objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {} #if CALC_EXACT_MALLOC_SIZE { @@ -8238,7 +8455,7 @@ objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size) #endif #define GC_MEMERROR(...) \ - ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : rb_memerror()) + ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0) #define TRY_WITH_GC(siz, expr) do { \ const gc_profile_record_flag gpr = \ @@ -8248,10 +8465,10 @@ objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size) GPR_FLAG_MALLOC; \ objspace_malloc_gc_stress(objspace); \ \ - if (RB_LIKELY((expr))) { \ + if (RB_LIKELY((expr))) { \ /* Success on 1st try */ \ } \ - else if (!garbage_collect_with_gvl(objspace, gpr)) { \ + else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \ /* @shyouhei thinks this doesn't happen */ \ GC_MEMERROR("TRY_WITH_GC: could not GC"); \ } \ @@ -8289,12 +8506,21 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) } #if CALC_EXACT_MALLOC_SIZE struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; +#if VERIFY_FREE_SIZE + if (!info->size) { + rb_bug("buffer %p has no recorded size. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr); + } + + if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) { + rb_bug("buffer %p freed with old_size=%zu, but was allocated with size=%zu", ptr, old_size, info->size - sizeof(struct malloc_obj_info)); + } +#endif ptr = info; old_size = info->size; #endif old_size = objspace_malloc_size(objspace, ptr, old_size); - objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE) { + objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE, true) { free(ptr); ptr = NULL; RB_DEBUG_COUNTER_INC(heap_xfree); @@ -8302,7 +8528,7 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) } void * -rb_gc_impl_malloc(void *objspace_ptr, size_t size) +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) { rb_objspace_t *objspace = objspace_ptr; check_malloc_not_in_gc(objspace, "malloc"); @@ -8312,11 +8538,12 @@ rb_gc_impl_malloc(void *objspace_ptr, size_t size) size = objspace_malloc_prepare(objspace, size); TRY_WITH_GC(size, mem = malloc(size)); RB_DEBUG_COUNTER_INC(heap_xmalloc); - return objspace_malloc_fixup(objspace, mem, size); + if (!mem) return mem; + return objspace_malloc_fixup(objspace, mem, size, gc_allowed); } void * -rb_gc_impl_calloc(void *objspace_ptr, size_t size) +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) { rb_objspace_t *objspace = objspace_ptr; @@ -8331,11 +8558,12 @@ rb_gc_impl_calloc(void *objspace_ptr, size_t size) size = objspace_malloc_prepare(objspace, size); TRY_WITH_GC(size, mem = calloc1(size)); - return objspace_malloc_fixup(objspace, mem, size); + if (!mem) return mem; + return objspace_malloc_fixup(objspace, mem, size, gc_allowed); } void * -rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size) +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) { rb_objspace_t *objspace = objspace_ptr; @@ -8343,7 +8571,7 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si void *mem; - if (!ptr) return rb_gc_impl_malloc(objspace, new_size); + if (!ptr) return rb_gc_impl_malloc(objspace, new_size, gc_allowed); /* * The behavior of realloc(ptr, 0) is implementation defined. @@ -8351,7 +8579,7 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm */ if (new_size == 0) { - if ((mem = rb_gc_impl_malloc(objspace, 0)) != NULL) { + if ((mem = rb_gc_impl_malloc(objspace, 0, gc_allowed)) != NULL) { /* * - OpenBSD's malloc(3) man page says that when 0 is passed, it * returns a non-NULL pointer to an access-protected memory page. @@ -8393,12 +8621,18 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; new_size += sizeof(struct malloc_obj_info); ptr = info; +#if VERIFY_FREE_SIZE + if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) { + rb_bug("buffer %p realloced with old_size=%zu, but was allocated with size=%zu", ptr, old_size, info->size - sizeof(struct malloc_obj_info)); + } +#endif old_size = info->size; } #endif old_size = objspace_malloc_size(objspace, ptr, old_size); TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size))); + if (!mem) return mem; new_size = objspace_malloc_size(objspace, mem, new_size); #if CALC_EXACT_MALLOC_SIZE @@ -8409,7 +8643,7 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si } #endif - objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC); + objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed); RB_DEBUG_COUNTER_INC(heap_xrealloc); return mem; @@ -8421,10 +8655,10 @@ rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) rb_objspace_t *objspace = objspace_ptr; if (diff > 0) { - objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC); + objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC, true); } else if (diff < 0) { - objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC); + objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC, true); } } @@ -8577,7 +8811,7 @@ gc_prof_timer_stop(rb_objspace_t *objspace) } } -#ifdef BUILDING_SHARED_GC +#ifdef BUILDING_MODULAR_GC # define RUBY_DTRACE_GC_HOOK(name) #else # define RUBY_DTRACE_GC_HOOK(name) \ @@ -8663,18 +8897,29 @@ gc_prof_set_heap_info(rb_objspace_t *objspace) { if (gc_prof_enabled(objspace)) { gc_profile_record *record = gc_prof_record(objspace); - size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace); - size_t total = objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT; + + /* Sum across all size pools since each has a different slot size. */ + size_t total = 0; + size_t use_size = 0; + size_t total_size = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count; + total += heap->total_slots; + use_size += heap_live * heap->slot_size; + total_size += heap->total_slots * heap->slot_size; + } #if GC_PROFILE_MORE_DETAIL + size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace); record->heap_use_pages = objspace->profile.heap_used_at_gc_start; record->heap_live_objects = live; record->heap_free_objects = total - live; #endif record->heap_total_objects = total; - record->heap_use_size = live * BASE_SLOT_SIZE; - record->heap_total_size = total * BASE_SLOT_SIZE; + record->heap_use_size = use_size; + record->heap_total_size = total_size; } } @@ -9041,6 +9286,12 @@ gc_profile_disable(VALUE _) return Qnil; } +void +rb_gc_verify_internal_consistency(void) +{ + gc_verify_internal_consistency(rb_gc_get_objspace()); +} + /* * call-seq: * GC.verify_internal_consistency -> nil @@ -9054,7 +9305,7 @@ gc_profile_disable(VALUE _) static VALUE gc_verify_internal_consistency_m(VALUE dummy) { - gc_verify_internal_consistency(rb_gc_get_objspace()); + rb_gc_verify_internal_consistency(); return Qnil; } @@ -9209,6 +9460,34 @@ gc_compact(VALUE self) #endif #if GC_CAN_COMPILE_COMPACTION +struct desired_compaction_pages_i_data { + rb_objspace_t *objspace; + size_t required_slots[HEAP_COUNT]; +}; + +static int +desired_compaction_pages_i(struct heap_page *page, void *data) +{ + struct desired_compaction_pages_i_data *tdata = data; + rb_objspace_t *objspace = tdata->objspace; + VALUE vstart = (VALUE)page->start; + VALUE vend = vstart + (VALUE)(page->total_slots * page->heap->slot_size); + + + for (VALUE v = vstart; v != vend; v += page->heap->slot_size) { + asan_unpoisoning_object(v) { + /* skip T_NONEs; they won't be moved */ + if (BUILTIN_TYPE(v) != T_NONE) { + rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, page->heap, v); + size_t dest_pool_idx = dest_pool - heaps; + tdata->required_slots[dest_pool_idx]++; + } + } + } + + return 0; +} + /* call-seq: * GC.verify_compaction_references(toward: nil, double_heap: false) -> hash * @@ -9247,7 +9526,7 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) /* Clear the heap. */ rb_gc_impl_start(objspace, true, true, true, false); - unsigned int lev = rb_gc_vm_lock(); + unsigned int lev = RB_GC_VM_LOCK(); { gc_rest(objspace); @@ -9262,16 +9541,14 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) /* Find out which pool has the most pages */ size_t max_existing_pages = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; max_existing_pages = MAX(max_existing_pages, heap->total_pages); } /* Add pages to each size pool so that compaction is guaranteed to move every object */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; size_t pages_to_add = 0; /* @@ -9285,9 +9562,9 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) * Step 2: Now add additional free pages to each size pool sufficient to hold all objects * that want to be in that size pool, whether moved into it or moved within it */ - objspace->heap_pages.allocatable_slots = desired_compaction.required_slots[i]; - while (objspace->heap_pages.allocatable_slots > 0) { - heap_page_allocate_and_initialize(objspace, size_pool, heap); + objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size; + while (objspace->heap_pages.allocatable_bytes > 0) { + heap_page_allocate_and_initialize(objspace, heap); } /* * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects @@ -9296,7 +9573,7 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) pages_to_add += 2; for (; pages_to_add > 0; pages_to_add--) { - heap_page_allocate_and_initialize_force(objspace, size_pool, heap); + heap_page_allocate_and_initialize_force(objspace, heap); } } } @@ -9305,7 +9582,7 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) objspace->rcompactor.compare_func = compare_free_slots; } } - rb_gc_vm_unlock(lev); + RB_GC_VM_UNLOCK(lev); rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true); @@ -9334,29 +9611,28 @@ rb_gc_impl_objspace_free(void *objspace_ptr) for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { heap_page_free(objspace, rb_darray_get(objspace->heap_pages.sorted, i)); } - rb_darray_free(objspace->heap_pages.sorted); + rb_darray_free_without_gc(objspace->heap_pages.sorted); heap_pages_lomem = 0; heap_pages_himem = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - SIZE_POOL_EDEN_HEAP(size_pool)->total_pages = 0; - SIZE_POOL_EDEN_HEAP(size_pool)->total_slots = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; + heap->total_pages = 0; + heap->total_slots = 0; } - st_free_table(objspace->id_to_obj_tbl); - st_free_table(objspace->obj_to_id_tbl); - free_stack_chunks(&objspace->mark_stack); mark_stack_free_cache(&objspace->mark_stack); - rb_darray_free(objspace->weak_references); + rb_darray_free_without_gc(objspace->weak_references); + +#ifdef MALLOC_COUNTERS_NEED_LOCK + rb_native_mutex_destroy(&objspace->malloc_counters.lock); +#endif free(objspace); } -void rb_gc_impl_mark(void *objspace_ptr, VALUE obj); - #if MALLOC_ALLOCATED_SIZE /* * call-seq: @@ -9370,7 +9646,8 @@ void rb_gc_impl_mark(void *objspace_ptr, VALUE obj); static VALUE gc_malloc_allocated_size(VALUE self) { - return UINT2NUM(rb_objspace.malloc_params.allocated_size); + rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace(); + return ULL2NUM(objspace->malloc_params.allocated_size); } /* @@ -9385,7 +9662,84 @@ gc_malloc_allocated_size(VALUE self) static VALUE gc_malloc_allocations(VALUE self) { - return UINT2NUM(rb_objspace.malloc_params.allocations); + rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace(); + return ULL2NUM(objspace->malloc_params.allocations); +} +#endif + +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + rb_objspace_t *objspace = objspace_ptr; + + objspace->fork_vm_lock_lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + rb_objspace_t *objspace = objspace_ptr; + + RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev); + objspace->fork_vm_lock_lev = 0; + + if (pid == 0) { /* child process */ + rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL); + } +} + +VALUE rb_ident_hash_new_with_size(st_index_t size); + +#if GC_DEBUG_STRESS_TO_CLASS +/* + * call-seq: + * GC.add_stress_to_class(class[, ...]) + * + * Raises NoMemoryError when allocating an instance of the given classes. + * + */ +static VALUE +rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + + if (!stress_to_class) { + set_stress_to_class(rb_ident_hash_new_with_size(argc)); + } + + for (int i = 0; i < argc; i++) { + VALUE klass = argv[i]; + rb_hash_aset(stress_to_class, klass, Qtrue); + } + + return self; +} + +/* + * call-seq: + * GC.remove_stress_to_class(class[, ...]) + * + * No longer raises NoMemoryError when allocating an instance of the + * given classes. + * + */ +static VALUE +rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + + if (stress_to_class) { + for (int i = 0; i < argc; ++i) { + rb_hash_delete(stress_to_class, argv[i]); + } + + if (rb_hash_size(stress_to_class) == 0) { + stress_to_class = 0; + } + } + + return Qnil; } #endif @@ -9404,47 +9758,42 @@ rb_gc_impl_objspace_init(void *objspace_ptr) gc_config_full_mark_set(TRUE); - objspace->flags.gc_stressful = RTEST(initial_stress); - objspace->gc_stress_mode = initial_stress; - objspace->flags.measure_gc = true; malloc_limit = gc_params.malloc_limit_min; +#ifdef MALLOC_COUNTERS_NEED_LOCK + rb_native_mutex_initialize(&objspace->malloc_counters.lock); +#endif objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace); if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) { rb_bug("Could not preregister postponed job for GC"); } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; + /* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit + * in at least one pool. In debug builds RVALUE_OVERHEAD can push this + * beyond the 48-byte pool into the 64-byte pool, which is fine. */ + GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]))); + + for (int i = 0; i < HEAP_COUNT; i++) { + rb_heap_t *heap = &heaps[i]; - size_pool->slot_size = (1 << i) * BASE_SLOT_SIZE; + heap->slot_size = pool_slot_sizes[i]; - ccan_list_head_init(&SIZE_POOL_EDEN_HEAP(size_pool)->pages); + ccan_list_head_init(&heap->pages); } - rb_darray_make(&objspace->heap_pages.sorted, 0); - rb_darray_make(&objspace->weak_references, 0); + init_size_to_heap_idx(); - // TODO: debug why on Windows Ruby crashes on boot when GC is on. -#ifdef _WIN32 - dont_gc_on(); -#endif + rb_darray_make_without_gc(&objspace->heap_pages.sorted, 0); + rb_darray_make_without_gc(&objspace->weak_references, 0); #if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP) /* Need to determine if we can use mmap at runtime. */ heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP; #endif - objspace->next_object_id = OBJ_ID_INITIAL; - objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type); - objspace->obj_to_id_tbl = st_init_numtable(); #if RGENGC_ESTIMATE_OLDMALLOC objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min; #endif - /* Set size pools allocatable pages. */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - /* Set the default value of size_pool_init_slots. */ - gc_params.size_pool_init_slots[i] = GC_HEAP_INIT_SLOTS; - } + gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES; init_mark_stack(&objspace->mark_stack); @@ -9457,13 +9806,18 @@ rb_gc_impl_init(void) { VALUE gc_constants = rb_hash_new(); rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD)); + /* Minimum slot size that fits a standard RVALUE */ + size_t rvalue_pool = 0; + for (size_t i = 0; i < HEAP_COUNT; i++) { + if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i]; break; } + } + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(SIZE_POOL_COUNT)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(size_pool_slot_size(SIZE_POOL_COUNT - 1))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(heap_slot_size(HEAP_COUNT - 1))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE)); if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) { rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue); @@ -9487,6 +9841,11 @@ rb_gc_impl_init(void) rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); } +#if GC_DEBUG_STRESS_TO_CLASS + rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1); + rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1); +#endif + /* internal methods */ rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0); diff --git a/gc/default/extconf.rb b/gc/default/extconf.rb new file mode 100644 index 0000000000..2940a4c962 --- /dev/null +++ b/gc/default/extconf.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "../extconf_base" + +create_gc_makefile("default") diff --git a/gc/extconf_base.rb b/gc/extconf_base.rb new file mode 100644 index 0000000000..2a224b9b0e --- /dev/null +++ b/gc/extconf_base.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "mkmf" + +srcdir = File.join(__dir__, "..") +$INCFLAGS << " -I#{srcdir}" + +$CPPFLAGS << " -DBUILDING_MODULAR_GC" + +append_cflags("-fPIC") + +def create_gc_makefile(name, &block) + create_makefile("librubygc.#{name}", &block) +end @@ -10,47 +10,153 @@ * first introduced for [Feature #20470]. */ #include "ruby/ruby.h" +#include "ruby/assert.h" +#include "ruby/thread_native.h" + +#ifndef VM_CHECK_MODE +# define VM_CHECK_MODE RUBY_DEBUG +#endif + +// From ractor_core.h +#ifndef RACTOR_CHECK_MODE +# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE) +#endif + +#if RACTOR_CHECK_MODE +void rb_ractor_setup_belonging(VALUE obj); + +struct rb_gc_obj_suffix { + uint32_t _ractor_belonging_id; +}; + +# define RB_GC_OBJ_HAS_SUFFIX 1 +# define RB_GC_OBJ_SUFFIX_SIZE (sizeof(struct rb_gc_obj_suffix)) +#else +# define RB_GC_OBJ_HAS_SUFFIX 0 +# define RB_GC_OBJ_SUFFIX_SIZE 0 +#endif + +struct rb_gc_vm_context { + rb_nativethread_lock_t lock; + + struct rb_execution_context_struct *ec; +}; + +typedef int (*vm_table_foreach_callback_func)(VALUE value, void *data); +typedef int (*vm_table_update_callback_func)(VALUE *value, void *data); + +enum rb_gc_vm_weak_tables { + RB_GC_VM_CI_TABLE, + RB_GC_VM_OVERLOADED_CME_TABLE, + RB_GC_VM_GLOBAL_SYMBOLS_TABLE, + RB_GC_VM_ID2REF_TABLE, + RB_GC_VM_GENERIC_FIELDS_TABLE, + RB_GC_VM_FROZEN_STRINGS_TABLE, + RB_GC_VM_WEAK_TABLE_COUNT +}; + +#define RB_GC_VM_LOCK() rb_gc_vm_lock(__FILE__, __LINE__) +#define RB_GC_VM_UNLOCK(lev) rb_gc_vm_unlock(lev, __FILE__, __LINE__) +#define RB_GC_CR_LOCK() rb_gc_cr_lock(__FILE__, __LINE__) +#define RB_GC_CR_UNLOCK(lev) rb_gc_cr_unlock(lev, __FILE__, __LINE__) +#define RB_GC_VM_LOCK_NO_BARRIER() rb_gc_vm_lock_no_barrier(__FILE__, __LINE__) +#define RB_GC_VM_UNLOCK_NO_BARRIER(lev) rb_gc_vm_unlock_no_barrier(lev, __FILE__, __LINE__) + +#if USE_MODULAR_GC +# define MODULAR_GC_FN +#else +// This takes advantage of internal linkage winning when appearing first. +// See C99 6.2.2p4. +# define MODULAR_GC_FN static +#endif + +#if USE_MODULAR_GC RUBY_SYMBOL_EXPORT_BEGIN -unsigned int rb_gc_vm_lock(void); -void rb_gc_vm_unlock(unsigned int lev); -unsigned int rb_gc_cr_lock(void); -void rb_gc_cr_unlock(unsigned int lev); -unsigned int rb_gc_vm_lock_no_barrier(void); -void rb_gc_vm_unlock_no_barrier(unsigned int lev); -void rb_gc_vm_barrier(void); -size_t rb_gc_obj_optimal_size(VALUE obj); -void rb_gc_mark_children(void *objspace, VALUE obj); -void rb_gc_update_object_references(void *objspace, VALUE obj); -void rb_gc_update_vm_references(void *objspace); -void rb_gc_event_hook(VALUE obj, rb_event_flag_t event); -void *rb_gc_get_objspace(void); +#endif + +// These functions cannot be defined as static because they are used by other +// files in Ruby. size_t rb_size_mul_or_raise(size_t x, size_t y, VALUE exc); -void rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data); -void rb_gc_set_pending_interrupt(void); -void rb_gc_unset_pending_interrupt(void); -bool rb_gc_obj_free(void *objspace, VALUE obj); -void rb_gc_mark_roots(void *objspace, const char **categoryp); -void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data); -bool rb_gc_multi_ractor_p(void); -void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data); void rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data); -void rb_obj_info_dump(VALUE obj); +const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj); const char *rb_obj_info(VALUE obj); -bool rb_gc_shutdown_call_finalizer_p(VALUE obj); -uint32_t rb_gc_get_shape(VALUE obj); -void rb_gc_set_shape(VALUE obj, uint32_t shape_id); -uint32_t rb_gc_rebuild_shape(VALUE obj, size_t size_pool_id); size_t rb_obj_memsize_of(VALUE obj); +bool ruby_free_at_exit_p(void); +void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data); +void rb_gc_verify_shareable(VALUE); + +MODULAR_GC_FN unsigned int rb_gc_vm_lock(const char *file, int line); +MODULAR_GC_FN void rb_gc_vm_unlock(unsigned int lev, const char *file, int line); +MODULAR_GC_FN unsigned int rb_gc_cr_lock(const char *file, int line); +MODULAR_GC_FN void rb_gc_cr_unlock(unsigned int lev, const char *file, int line); +MODULAR_GC_FN unsigned int rb_gc_vm_lock_no_barrier(const char *file, int line); +MODULAR_GC_FN void rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line); +MODULAR_GC_FN void rb_gc_vm_barrier(void); +MODULAR_GC_FN size_t rb_gc_obj_optimal_size(VALUE obj); +MODULAR_GC_FN void rb_gc_mark_children(void *objspace, VALUE obj); +MODULAR_GC_FN void rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, vm_table_update_callback_func update_callback, void *data, bool weak_only, enum rb_gc_vm_weak_tables table); +MODULAR_GC_FN void rb_gc_update_object_references(void *objspace, VALUE obj); +MODULAR_GC_FN void rb_gc_update_vm_references(void *objspace); +MODULAR_GC_FN void rb_gc_event_hook(VALUE obj, rb_event_flag_t event); +MODULAR_GC_FN void *rb_gc_get_objspace(void); +MODULAR_GC_FN void rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data); +MODULAR_GC_FN void rb_gc_set_pending_interrupt(void); +MODULAR_GC_FN void rb_gc_unset_pending_interrupt(void); +MODULAR_GC_FN void rb_gc_obj_free_vm_weak_references(VALUE obj); +MODULAR_GC_FN bool rb_gc_obj_free(void *objspace, VALUE obj); +MODULAR_GC_FN void rb_gc_save_machine_context(void); +MODULAR_GC_FN void rb_gc_mark_roots(void *objspace, const char **categoryp); +MODULAR_GC_FN void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data); +MODULAR_GC_FN bool rb_gc_multi_ractor_p(void); +MODULAR_GC_FN bool rb_gc_shutdown_call_finalizer_p(VALUE obj); +MODULAR_GC_FN void rb_gc_obj_changed_pool(VALUE obj, size_t heap_id); +MODULAR_GC_FN void rb_gc_prepare_heap_process_object(VALUE obj); +MODULAR_GC_FN bool rb_memerror_reentered(void); +MODULAR_GC_FN bool rb_obj_id_p(VALUE); +MODULAR_GC_FN void rb_gc_before_updating_jit_code(void); +MODULAR_GC_FN void rb_gc_after_updating_jit_code(void); +MODULAR_GC_FN bool rb_gc_obj_shareable_p(VALUE); +MODULAR_GC_FN void rb_gc_rp(VALUE); +MODULAR_GC_FN void rb_gc_handle_weak_references(VALUE obj); +MODULAR_GC_FN bool rb_gc_obj_needs_cleanup_p(VALUE obj); + +#if USE_MODULAR_GC +MODULAR_GC_FN bool rb_gc_event_hook_required_p(rb_event_flag_t event); +MODULAR_GC_FN void *rb_gc_get_ractor_newobj_cache(void); +MODULAR_GC_FN void rb_gc_initialize_vm_context(struct rb_gc_vm_context *context); +MODULAR_GC_FN void rb_gc_move_obj_during_marking(VALUE from, VALUE to); +MODULAR_GC_FN void rb_gc_print_backtrace(); +#endif + +#if USE_MODULAR_GC RUBY_SYMBOL_EXPORT_END +#endif void rb_ractor_finish_marking(void); // -------------------Private section begin------------------------ // Functions in this section are private to the default GC and gc.c +#ifdef BUILDING_MODULAR_GC +RBIMPL_WARNING_PUSH() +RBIMPL_WARNING_IGNORED(-Wunused-function) +#endif + +/* RGENGC_CHECK_MODE + * 0: disable all assertions + * 1: enable assertions (to debug RGenGC) + * 2: enable internal consistency check at each GC (for debugging) + * 3: enable internal consistency check at each GC steps (for debugging) + * 4: enable liveness check + * 5: show all references + */ +#ifndef RGENGC_CHECK_MODE +# define RGENGC_CHECK_MODE 0 +#endif + #ifndef GC_ASSERT -# define GC_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr) +# define GC_ASSERT(expr, ...) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__)) #endif static int @@ -89,6 +195,14 @@ gc_mark_tbl_no_pin_i(st_data_t key, st_data_t value, st_data_t data) } static int +gc_mark_set_no_pin_i(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_mark_movable((VALUE)key); + + return ST_CONTINUE; +} + +static int hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error) { if (rb_gc_location((VALUE)key) != (VALUE)key) { @@ -105,12 +219,14 @@ hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error) static int hash_replace_ref(st_data_t *key, st_data_t *value, st_data_t argp, int existing) { - if (rb_gc_location((VALUE)*key) != (VALUE)*key) { - *key = rb_gc_location((VALUE)*key); + VALUE new_key = rb_gc_location((VALUE)*key); + if (new_key != (VALUE)*key) { + *key = new_key; } - if (rb_gc_location((VALUE)*value) != (VALUE)*value) { - *value = rb_gc_location((VALUE)*value); + VALUE new_value = rb_gc_location((VALUE)*value); + if (new_value != (VALUE)*value) { + *value = new_value; } return ST_CONTINUE; @@ -168,6 +284,10 @@ type_sym(size_t type) default: return SIZET2NUM(type); break; } } + +#ifdef BUILDING_MODULAR_GC +RBIMPL_WARNING_POP() +#endif // -------------------Private section end------------------------ #endif diff --git a/gc/gc_impl.h b/gc/gc_impl.h index 12527845f7..d9e44cc66d 100644 --- a/gc/gc_impl.h +++ b/gc/gc_impl.h @@ -10,7 +10,18 @@ */ #include "ruby/ruby.h" -// `GC_IMPL_FN` is an implementation detail of `!USE_SHARED_GC` builds +#ifndef RB_GC_OBJECT_METADATA_ENTRY_DEFINED +# define RB_GC_OBJECT_METADATA_ENTRY_DEFINED +struct rb_gc_object_metadata_entry { + ID name; + VALUE val; +}; +#endif + +#ifdef BUILDING_MODULAR_GC +# define GC_IMPL_FN +#else +// `GC_IMPL_FN` is an implementation detail of `!USE_MODULAR_GC` builds // to have the default GC in the same translation unit as gc.c for // the sake of optimizer visibility. It expands to nothing unless // you're the default GC. @@ -18,24 +29,20 @@ // For the default GC, do not copy-paste this when implementing // these functions. This takes advantage of internal linkage winning // when appearing first. See C99 6.2.2p4. -#ifdef RB_AMALGAMATED_DEFAULT_GC # define GC_IMPL_FN static -#else -# define GC_IMPL_FN #endif // Bootup GC_IMPL_FN void *rb_gc_impl_objspace_alloc(void); GC_IMPL_FN void rb_gc_impl_objspace_init(void *objspace_ptr); -GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr); -GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr); -GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache); +GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor); GC_IMPL_FN void rb_gc_impl_set_params(void *objspace_ptr); GC_IMPL_FN void rb_gc_impl_init(void); -GC_IMPL_FN void rb_gc_impl_initial_stress_set(VALUE flag); -GC_IMPL_FN size_t *rb_gc_impl_size_pool_sizes(void *objspace_ptr); +GC_IMPL_FN size_t *rb_gc_impl_heap_sizes(void *objspace_ptr); // Shutdown GC_IMPL_FN void rb_gc_impl_shutdown_free_objects(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache); // GC GC_IMPL_FN void rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact); GC_IMPL_FN bool rb_gc_impl_during_gc_p(void *objspace_ptr); @@ -46,16 +53,29 @@ GC_IMPL_FN bool rb_gc_impl_gc_enabled_p(void *objspace_ptr); GC_IMPL_FN void rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag); GC_IMPL_FN VALUE rb_gc_impl_stress_get(void *objspace_ptr); GC_IMPL_FN VALUE rb_gc_impl_config_get(void *objspace_ptr); -GC_IMPL_FN VALUE rb_gc_impl_config_set(void *objspace_ptr, VALUE hash); +GC_IMPL_FN void rb_gc_impl_config_set(void *objspace_ptr, VALUE hash); +GC_IMPL_FN struct rb_gc_vm_context *rb_gc_impl_get_vm_context(void *objspace_ptr); // Object allocation -GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size); +GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size); GC_IMPL_FN size_t rb_gc_impl_obj_slot_size(VALUE obj); -GC_IMPL_FN size_t rb_gc_impl_size_pool_id_for_size(void *objspace_ptr, size_t size); +GC_IMPL_FN size_t rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size); GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size); // Malloc -GC_IMPL_FN void *rb_gc_impl_malloc(void *objspace_ptr, size_t size); -GC_IMPL_FN void *rb_gc_impl_calloc(void *objspace_ptr, size_t size); -GC_IMPL_FN void *rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size); +/* + * BEWARE: These functions may or may not run under GVL. + * + * You might want to make them thread-safe. + * Garbage collecting inside is possible if and only if you + * already have GVL. Also raising exceptions without one is a + * total disaster. + * + * When you absolutely cannot allocate the requested amount of + * memory just return NULL (with appropriate errno set). + * The caller side takes care of that situation. + */ +GC_IMPL_FN void *rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed); +GC_IMPL_FN void *rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed); +GC_IMPL_FN void *rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed); GC_IMPL_FN void rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size); GC_IMPL_FN void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff); // Marking @@ -63,9 +83,11 @@ GC_IMPL_FN void rb_gc_impl_mark(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr); GC_IMPL_FN void rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj); -GC_IMPL_FN void rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr); -GC_IMPL_FN void rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr); +// Weak references +GC_IMPL_FN void rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj); +GC_IMPL_FN bool rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj); // Compaction +GC_IMPL_FN void rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj); GC_IMPL_FN bool rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj); GC_IMPL_FN VALUE rb_gc_impl_location(void *objspace_ptr, VALUE value); // Write barriers @@ -81,19 +103,20 @@ GC_IMPL_FN VALUE rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALU GC_IMPL_FN void rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj); GC_IMPL_FN void rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr); -// Object ID -GC_IMPL_FN VALUE rb_gc_impl_object_id(void *objspace_ptr, VALUE obj); -GC_IMPL_FN VALUE rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id); +// Forking +GC_IMPL_FN void rb_gc_impl_before_fork(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid); // Statistics -GC_IMPL_FN VALUE rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag); -GC_IMPL_FN VALUE rb_gc_impl_get_measure_total_time(void *objspace_ptr); -GC_IMPL_FN VALUE rb_gc_impl_get_profile_total_time(void *objspace_ptr); +GC_IMPL_FN void rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag); +GC_IMPL_FN bool rb_gc_impl_get_measure_total_time(void *objspace_ptr); +GC_IMPL_FN unsigned long long rb_gc_impl_get_total_time(void *objspace_ptr); GC_IMPL_FN size_t rb_gc_impl_gc_count(void *objspace_ptr); GC_IMPL_FN VALUE rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key); -GC_IMPL_FN size_t rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym); -GC_IMPL_FN size_t rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym); +GC_IMPL_FN VALUE rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym); +GC_IMPL_FN VALUE rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym); +GC_IMPL_FN const char *rb_gc_impl_active_gc_name(void); // Miscellaneous -GC_IMPL_FN size_t rb_gc_impl_obj_flags(void *objspace_ptr, VALUE obj, ID* flags, size_t max); +GC_IMPL_FN struct rb_gc_object_metadata_entry *rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj); GC_IMPL_FN bool rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr); GC_IMPL_FN bool rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event); diff --git a/gc/mmtk/.gitignore b/gc/mmtk/.gitignore new file mode 100644 index 0000000000..eb5a316cbd --- /dev/null +++ b/gc/mmtk/.gitignore @@ -0,0 +1 @@ +target diff --git a/gc/mmtk/Cargo.lock b/gc/mmtk/Cargo.lock new file mode 100644 index 0000000000..910048fa80 --- /dev/null +++ b/gc/mmtk/Cargo.lock @@ -0,0 +1,1108 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-traits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f750b93bd1b739cf9ddf85f8fe7c97a4a62c60ccf8b6f232514bd9103bedc" +dependencies = [ + "cfg-if", + "rustc_version", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" +dependencies = [ + "git2", +] + +[[package]] +name = "bytemuck" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "cc" +version = "1.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "delegate" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b6483c2bbed26f97861cf57651d4f2b731964a28cd2257f934a4b452480d21" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "downcast-rs" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279259b0ac81c89d11c290495fdcfa96ea3643b7df311c138b6fe8ca5237f0f8" +dependencies = [ + "idna_mapping", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna_mapping" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c13906586a4b339310541a274dd927aff6fcbb5b8e3af90634c4b31681c792" +dependencies = [ + "unicode-joining-type", +] + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.1", + "libc", + "windows-sys", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "jiff" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libgit2-sys" +version = "0.18.3+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mmtk" +version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791" +dependencies = [ + "atomic", + "atomic-traits", + "atomic_refcell", + "built", + "bytemuck", + "bytemuck_derive", + "cfg-if", + "crossbeam", + "delegate", + "downcast-rs", + "enum-map", + "env_logger", + "idna_adapter", + "is-terminal", + "itertools", + "lazy_static", + "libc", + "log", + "memoffset", + "mmtk-macros", + "num-traits", + "num_cpus", + "portable-atomic", + "probe", + "rayon-core", + "regex", + "rustversion", + "spin", + "static_assertions", + "strum", + "strum_macros", + "sysinfo 0.33.1", +] + +[[package]] +name = "mmtk-macros" +version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "mmtk_ruby" +version = "0.1.0" +dependencies = [ + "atomic_refcell", + "env_logger", + "libc", + "log", + "mmtk", + "once_cell", + "probe", + "sysinfo 0.32.1", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "probe" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e2d2444b730c8f027344c60f9e1f1554d7a3342df9bdd425142ed119a6e5a3" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" + +[[package]] +name = "strum_macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.101", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-joining-type" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d00a78170970967fdb83f9d49b92f959ab2bb829186b113e4f4604ad98e180" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] diff --git a/gc/mmtk/Cargo.toml b/gc/mmtk/Cargo.toml new file mode 100644 index 0000000000..d856122900 --- /dev/null +++ b/gc/mmtk/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "mmtk_ruby" +version = "0.1.0" +authors = [] +edition = "2021" + +[lib] +name = "mmtk_ruby" +crate-type = ["cdylib", "staticlib"] + +[profile.release] +lto = true + +[dependencies] +libc = "0.2" +log = "0.4.14" +env_logger = "0.11.3" +once_cell = "1.17.0" +atomic_refcell = "0.1.9" +probe = "0.5" +sysinfo = "0.32.0" + +[dependencies.mmtk] +features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"] + +# Uncomment the following lines to use mmtk-core from the official repository. +git = "https://github.com/mmtk/mmtk-core.git" +rev = "c6317a3f1c262e33fc2e427e4cc999c17bcc4791" + +# Uncomment the following line to use mmtk-core from a local repository. +# path = "../../../mmtk-core" + +[features] +default = [] + +# When moving an object, clear its original copy. +clear_old_copy = [] + +# Enable extra assertions in release build. For debugging. +extra_assert = [] + +[workspace] diff --git a/gc/mmtk/cbindgen.toml b/gc/mmtk/cbindgen.toml new file mode 100644 index 0000000000..b99c30efc8 --- /dev/null +++ b/gc/mmtk/cbindgen.toml @@ -0,0 +1,36 @@ +language = "C" + +include_guard = "MMTK_H" + +autogen_warning = "/* Warning, this file is autogenerated by cbindgen from the mmtk-ruby repository. Don't modify this manually. */" + +tab_width = 4 + +usize_is_size_t = true + +after_includes = """ + +typedef struct MMTk_Builder MMTk_Builder; +typedef struct MMTk_Mutator MMTk_Mutator; + +typedef struct MMTk_ractor_cache *MMTk_VMThread; +typedef struct MMTk_ractor_cache *MMTk_VMMutatorThread; +typedef struct MMTk_GCThreadTLS *MMTk_VMWorkerThread; +typedef void *MMTk_Address; +typedef void *MMTk_ObjectReference; +typedef void *MMTk_NullableObjectReference; +typedef uint32_t MMTk_AllocationSemantics; + +typedef struct MMTk_BumpPointer { + uintptr_t cursor; + uintptr_t limit; +} MMTk_BumpPointer; +""" + +[export] +exclude = ["RubyMutator"] +prefix = "MMTk_" + +[export.rename] +"MMTKBuilder" = "Builder" +"RubyMutator" = "Mutator" diff --git a/gc/mmtk/depend b/gc/mmtk/depend new file mode 100644 index 0000000000..77b229af36 --- /dev/null +++ b/gc/mmtk/depend @@ -0,0 +1,18 @@ +$(TARGET_SO): $(MMTK_BUILD)/$(LIBMMTK_RUBY) + +# Add the `libmmtk_ruby.a` target to run `cargo build` + +release/$(LIBMMTK_RUBY) debug/$(LIBMMTK_RUBY): $(RUSTSRCS) $(srcdir)/Cargo.toml $(srcdir)/Cargo.toml + +release/$(LIBMMTK_RUBY): + CARGO_TARGET_DIR="." cargo build --manifest-path=$(srcdir)/Cargo.toml --release + +debug/$(LIBMMTK_RUBY): + CARGO_TARGET_DIR="." cargo build --manifest-path=$(srcdir)/Cargo.toml + +clean: clean-mmtk + +.PHONY: clean-mmtk +clean-mmtk: + -$(Q)$(RM_RF) debug release + -$(Q)$(RM) .rustc_info.json diff --git a/gc/mmtk/extconf.rb b/gc/mmtk/extconf.rb new file mode 100644 index 0000000000..c0e788037e --- /dev/null +++ b/gc/mmtk/extconf.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require_relative "../extconf_base" + +# Statically link `libmmtk_ruby.a` +$LIBS << " $(MMTK_BUILD)/$(LIBMMTK_RUBY)" + +rustsrcs = Dir.glob("src/*.rs", base: __dir__).map {|s| "$(srcdir)/#{s}"} + +create_gc_makefile("mmtk") do |makefile| + [ + *makefile, + + <<~MAKEFILE, + MMTK_BUILD = debug + LIBMMTK_RUBY = libmmtk_ruby.#$LIBEXT + RUSTSRCS = #{rustsrcs.join(" \\\n\t ")} + + ifeq ($(MMTK_BUILD), debug) + CPPFLAGS += -DMMTK_DEBUG + endif + MAKEFILE + ] +end diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c new file mode 100644 index 0000000000..8be69b4fe6 --- /dev/null +++ b/gc/mmtk/mmtk.c @@ -0,0 +1,1658 @@ +#include <pthread.h> +#include <stdbool.h> + +#include "ruby/assert.h" +#include "ruby/atomic.h" +#include "ruby/debug.h" + +#include "gc/gc.h" +#include "gc/gc_impl.h" +#include "gc/mmtk/mmtk.h" + +#include "ccan/list/list.h" +#include "darray.h" + +#ifdef __APPLE__ +#include <sys/sysctl.h> +#endif + +struct objspace { + bool measure_gc_time; + bool gc_stress; + + size_t gc_count; + size_t moving_gc_count; + size_t total_gc_time; + size_t total_allocated_objects; + + st_table *finalizer_table; + struct MMTk_final_job *finalizer_jobs; + rb_postponed_job_handle_t finalizer_postponed_job; + + struct ccan_list_head ractor_caches; + unsigned long live_ractor_cache_count; + + pthread_mutex_t mutex; + rb_atomic_t mutator_blocking_count; + bool world_stopped; + pthread_cond_t cond_world_stopped; + pthread_cond_t cond_world_started; + size_t start_the_world_count; + + pthread_mutex_t event_hook_mutex; + + struct { + bool gc_thread_crashed; + char crash_msg[256]; + } crash_context; + + struct rb_gc_vm_context vm_context; + + unsigned int fork_hook_vm_lock_lev; +}; + +#define OBJ_FREE_BUF_CAPACITY 128 + +struct MMTk_ractor_cache { + struct ccan_list_node list_node; + + MMTk_Mutator *mutator; + bool gc_mutator_p; + + MMTk_BumpPointer *bump_pointer; + + MMTk_ObjectReference obj_free_parallel_buf[OBJ_FREE_BUF_CAPACITY]; + size_t obj_free_parallel_count; + MMTk_ObjectReference obj_free_non_parallel_buf[OBJ_FREE_BUF_CAPACITY]; + size_t obj_free_non_parallel_count; +}; + +struct MMTk_final_job { + struct MMTk_final_job *next; + enum { + MMTK_FINAL_JOB_DFREE, + MMTK_FINAL_JOB_FINALIZE, + } kind; + union { + struct { + void (*func)(void *); + void *data; + } dfree; + struct { + /* HACK: we store the object ID on the 0th element of this array. */ + VALUE finalizer_array; + } finalize; + } as; +}; + +#ifdef RB_THREAD_LOCAL_SPECIFIER +RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls; + +RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object; +#else +# error We currently need language-supported TLS +#endif + +#ifdef MMTK_DEBUG +# define MMTK_ASSERT(expr, ...) RUBY_ASSERT_ALWAYS(expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__)) +#else +# define MMTK_ASSERT(expr, ...) ((void)0) +#endif + +#include <pthread.h> + +static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); + +static void +rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls) +{ + rb_mmtk_gc_thread_tls = gc_thread_tls; +} + +static bool +rb_mmtk_is_mutator(void) +{ + return ruby_native_thread_p(); +} + +static void +rb_mmtk_stop_the_world(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + while (!objspace->world_stopped) { + pthread_cond_wait(&objspace->cond_world_stopped, &objspace->mutex); + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void +rb_mmtk_resume_mutators(bool current_gc_may_move) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + objspace->world_stopped = false; + objspace->gc_count++; + if (current_gc_may_move) objspace->moving_gc_count++; + pthread_cond_broadcast(&objspace->cond_world_started); + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache); + +static void +rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + size_t starting_gc_count = objspace->gc_count; + RUBY_ATOMIC_INC(objspace->mutator_blocking_count); + int lock_lev = RB_GC_VM_LOCK(); + RUBY_ATOMIC_DEC(objspace->mutator_blocking_count); + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + if (objspace->gc_count == starting_gc_count) { + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START); + + rb_gc_initialize_vm_context(&objspace->vm_context); + + mutator->gc_mutator_p = true; + + struct timespec gc_start_time; + if (objspace->measure_gc_time) { + clock_gettime(CLOCK_MONOTONIC, &gc_start_time); + } + + rb_gc_save_machine_context(); + + rb_gc_vm_barrier(); + + struct MMTk_ractor_cache *rc; + ccan_list_for_each(&objspace->ractor_caches, rc, list_node) { + mmtk_flush_obj_free_buffer(rc); + } + + objspace->world_stopped = true; + + pthread_cond_broadcast(&objspace->cond_world_stopped); + + // Wait for GC end + while (objspace->world_stopped) { + pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex); + } + + if (RB_UNLIKELY(objspace->crash_context.gc_thread_crashed)) { + rb_bug("%s", objspace->crash_context.crash_msg); + } + + if (objspace->measure_gc_time) { + struct timespec gc_end_time; + clock_gettime(CLOCK_MONOTONIC, &gc_end_time); + + objspace->total_gc_time += + (gc_end_time.tv_sec - gc_start_time.tv_sec) * (1000 * 1000 * 1000) + + (gc_end_time.tv_nsec - gc_start_time.tv_nsec); + } + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } + RB_GC_VM_UNLOCK(lock_lev); +} + +static void +rb_mmtk_before_updating_jit_code(void) +{ + rb_gc_before_updating_jit_code(); +} + +static void +rb_mmtk_after_updating_jit_code(void) +{ + rb_gc_after_updating_jit_code(); +} + +static size_t +rb_mmtk_number_of_mutators(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + return objspace->live_ractor_cache_count; +} + +static void +rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data) +{ + struct objspace *objspace = rb_gc_get_objspace(); + struct MMTk_ractor_cache *ractor_cache; + + ccan_list_for_each(&objspace->ractor_caches, ractor_cache, list_node) { + visit_mutator(ractor_cache->mutator, data); + } +} + +static void +rb_mmtk_scan_gc_roots(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + rb_gc_mark_roots(objspace, NULL); +} + +static int +pin_value(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); + + return ST_CONTINUE; +} + +static void +rb_mmtk_scan_objspace(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + if (objspace->finalizer_table != NULL) { + st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace); + } + + struct MMTk_final_job *job = objspace->finalizer_jobs; + while (job != NULL) { + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + break; + case MMTK_FINAL_JOB_FINALIZE: + rb_gc_impl_mark(objspace, job->as.finalize.finalizer_array); + break; + default: + rb_bug("rb_mmtk_scan_objspace: unknown final job type %d", job->kind); + } + + job = job->next; + } +} + +static void +rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to) +{ + rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to); +} + +static void +rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object) +{ + VALUE object = (VALUE)mmtk_object; + + if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) { + marking_parent_object = object; + rb_gc_update_object_references(rb_gc_get_objspace(), object); + marking_parent_object = 0; + } +} + +static void +rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object) +{ + marking_parent_object = (VALUE)object; + rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object); + marking_parent_object = 0; +} + +static void +rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving) +{ + VALUE object = (VALUE)mmtk_object; + + marking_parent_object = object; + + rb_gc_handle_weak_references(object); + + if (moving) { + rb_gc_update_object_references(rb_gc_get_objspace(), object); + } + + marking_parent_object = 0; +} + +static void +rb_mmtk_call_obj_free(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + struct objspace *objspace = rb_gc_get_objspace(); + + if (RB_UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_FREEOBJ))) { + pthread_mutex_lock(&objspace->event_hook_mutex); + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); + pthread_mutex_unlock(&objspace->event_hook_mutex); + } + + if (RB_UNLIKELY(rb_gc_obj_needs_cleanup_p(obj))) { + rb_gc_obj_free(objspace, obj); + } + +#ifdef MMTK_DEBUG + memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj)); +#endif +} + +static size_t +rb_mmtk_vm_live_bytes(void) +{ + return 0; +} + +static void +make_final_job(struct objspace *objspace, VALUE obj, VALUE table) +{ + MMTK_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY); + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->next = objspace->finalizer_jobs; + job->kind = MMTK_FINAL_JOB_FINALIZE; + job->as.finalize.finalizer_array = table; + + objspace->finalizer_jobs = job; +} + +static int +rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data, int error) +{ + MMTK_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value)); + MMTK_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY); + + struct objspace *objspace = (struct objspace *)data; + + if (mmtk_is_reachable((MMTk_ObjectReference)key)) { + VALUE new_key_location = rb_mmtk_call_object_closure((VALUE)key, false); + + MMTK_ASSERT(RB_FL_TEST(new_key_location, RUBY_FL_FINALIZE)); + + if (new_key_location != key) { + return ST_REPLACE; + } + } + else { + make_final_job(objspace, (VALUE)key, (VALUE)value); + + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static int +rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_data_t data, int existing) +{ + *key = rb_mmtk_call_object_closure((VALUE)*key, false); + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_finalizer_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + st_foreach_with_replace( + objspace->finalizer_table, + rb_mmtk_update_finalizer_table_i, + rb_mmtk_update_finalizer_table_replace_i, + (st_data_t)objspace + ); +} + +static int +rb_mmtk_global_tables_count(void) +{ + return RB_GC_VM_WEAK_TABLE_COUNT; +} + +static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); + +static int +rb_mmtk_update_global_tables_i(VALUE val, void *data) +{ + if (!mmtk_is_reachable((MMTk_ObjectReference)val)) { + return ST_DELETE; + } + + // TODO: check only if in moving GC + if (rb_mmtk_call_object_closure(val, false) != val) { + return ST_REPLACE; + } + + return ST_CONTINUE; +} + +static int +rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data) +{ + // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice + *ptr = rb_mmtk_call_object_closure(*ptr, false); + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_global_tables(int table, bool moving) +{ + MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT); + + rb_gc_vm_weak_table_foreach( + rb_mmtk_update_global_tables_i, + rb_mmtk_update_global_tables_replace_i, + NULL, + !moving, + (enum rb_gc_vm_weak_tables)table + ); +} + +static bool +rb_mmtk_special_const_p(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + + return RB_SPECIAL_CONST_P(obj); +} + +RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2) +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_gc_thread_bug(const char *msg, ...) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + objspace->crash_context.gc_thread_crashed = true; + + va_list args; + va_start(args, msg); + vsnprintf(objspace->crash_context.crash_msg, sizeof(objspace->crash_context.crash_msg), msg, args); + va_end(args); + + fprintf(stderr, "-- GC thread backtrace " + "-------------------------------------------\n"); + rb_gc_print_backtrace(); + fprintf(stderr, "\n"); + + rb_mmtk_resume_mutators(false); + + sleep(5); + + rb_bug("rb_mmtk_gc_thread_bug"); +} + +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_gc_thread_panic_handler(void) +{ + rb_mmtk_gc_thread_bug("MMTk GC thread panicked"); +} + +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_mutator_thread_panic_handler(void) +{ + rb_bug("Ruby mutator thread panicked"); +} + +// Bootup +MMTk_RubyUpcalls ruby_upcalls = { + rb_mmtk_init_gc_worker_thread, + rb_mmtk_is_mutator, + rb_mmtk_stop_the_world, + rb_mmtk_resume_mutators, + rb_mmtk_block_for_gc, + rb_mmtk_before_updating_jit_code, + rb_mmtk_after_updating_jit_code, + rb_mmtk_number_of_mutators, + rb_mmtk_get_mutators, + rb_mmtk_scan_gc_roots, + rb_mmtk_scan_objspace, + rb_mmtk_move_obj_during_marking, + rb_mmtk_update_object_references, + rb_mmtk_call_gc_mark_children, + rb_mmtk_handle_weak_references, + rb_mmtk_call_obj_free, + rb_mmtk_vm_live_bytes, + rb_mmtk_update_global_tables, + rb_mmtk_global_tables_count, + rb_mmtk_update_finalizer_table, + rb_mmtk_special_const_p, + rb_mmtk_mutator_thread_panic_handler, + rb_mmtk_gc_thread_panic_handler, +}; + +// Use max 80% of the available memory by default for MMTk +#define RB_MMTK_HEAP_LIMIT_PERC 80 +#define RB_MMTK_DEFAULT_HEAP_MIN (1024 * 1024) +#define RB_MMTK_DEFAULT_HEAP_MAX (rb_mmtk_system_physical_memory() / 100 * RB_MMTK_HEAP_LIMIT_PERC) + +enum mmtk_heap_mode { + RB_MMTK_DYNAMIC_HEAP, + RB_MMTK_FIXED_HEAP +}; + +MMTk_Builder * +rb_mmtk_builder_init(void) +{ + MMTk_Builder *builder = mmtk_builder_default(); + return builder; +} + +void * +rb_gc_impl_objspace_alloc(void) +{ + MMTk_Builder *builder = rb_mmtk_builder_init(); + MMTk_RubyBindingOptions binding_options = { + .suffix_size = RB_GC_OBJ_SUFFIX_SIZE, + }; + mmtk_init_binding(builder, &binding_options, &ruby_upcalls); + + return calloc(1, sizeof(struct objspace)); +} + +static void gc_run_finalizers(void *data); + +void +rb_gc_impl_objspace_init(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + objspace->measure_gc_time = true; + + objspace->finalizer_table = st_init_numtable(); + objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace); + + ccan_list_head_init(&objspace->ractor_caches); + + objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; + objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + + objspace->event_hook_mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; +} + +void +rb_gc_impl_objspace_free(void *objspace_ptr) +{ + free(objspace_ptr); +} + +void * +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) +{ + struct objspace *objspace = objspace_ptr; + if (objspace->live_ractor_cache_count == 0) { + mmtk_initialize_collection(ractor); + } + objspace->live_ractor_cache_count++; + + struct MMTk_ractor_cache *cache = calloc(1, sizeof(struct MMTk_ractor_cache)); + ccan_list_add(&objspace->ractor_caches, &cache->list_node); + + cache->mutator = mmtk_bind_mutator(cache); + cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator); + + return cache; +} + +void +rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache_ptr) +{ + struct objspace *objspace = objspace_ptr; + struct MMTk_ractor_cache *cache = cache_ptr; + + ccan_list_del(&cache->list_node); + + mmtk_flush_obj_free_buffer(cache); + + if (ruby_free_at_exit_p()) { + MMTK_ASSERT(objspace->live_ractor_cache_count > 0); + } + else { + MMTK_ASSERT(objspace->live_ractor_cache_count > 1); + } + + objspace->live_ractor_cache_count--; + + mmtk_destroy_mutator(cache->mutator); +} + +void rb_gc_impl_set_params(void *objspace_ptr) { } + +static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; } + +#if SIZEOF_VALUE >= 8 +#define MMTK_HEAP_COUNT 12 +#define MMTK_MAX_OBJ_SIZE 1024 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 40, 64, 80, 96, 128, 160, 256, 512, 640, 768, MMTK_MAX_OBJ_SIZE, 0 +}; +#else +#define MMTK_HEAP_COUNT 5 +#define MMTK_MAX_OBJ_SIZE 512 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0 +}; +#endif + +void +rb_gc_impl_init(void) +{ + VALUE gc_constants = rb_hash_new(); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); + // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0)); + OBJ_FREEZE(gc_constants); + rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants); + + // no-ops for compatibility + rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0); + + rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); +} + +size_t * +rb_gc_impl_heap_sizes(void *objspace_ptr) +{ + return heap_sizes; +} + +int +rb_mmtk_obj_free_iter_wrapper(VALUE obj, void *data) +{ + struct objspace *objspace = data; + + if (!RB_TYPE_P(obj, T_NONE)) { + rb_gc_obj_free_vm_weak_references(obj); + rb_gc_obj_free(objspace, obj); + } + + return 0; +} + +// Shutdown +static void each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data); + +void +rb_gc_impl_shutdown_free_objects(void *objspace_ptr) +{ + mmtk_set_gc_enabled(false); + each_object(objspace_ptr, rb_mmtk_obj_free_iter_wrapper, objspace_ptr); + mmtk_set_gc_enabled(true); +} + +// GC +void +rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact) +{ + mmtk_handle_user_collection_request(rb_gc_get_ractor_newobj_cache(), true, full_mark); +} + +bool +rb_gc_impl_during_gc_p(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + return objspace->world_stopped; +} + +static void +rb_gc_impl_prepare_heap_i(MMTk_ObjectReference obj, void *d) +{ + rb_gc_prepare_heap_process_object((VALUE)obj); +} + +void +rb_gc_impl_prepare_heap(void *objspace_ptr) +{ + mmtk_enumerate_objects(rb_gc_impl_prepare_heap_i, NULL); +} + +void +rb_gc_impl_gc_enable(void *objspace_ptr) +{ + mmtk_set_gc_enabled(true); +} + +void +rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc) +{ + mmtk_set_gc_enabled(false); +} + +bool +rb_gc_impl_gc_enabled_p(void *objspace_ptr) +{ + return mmtk_gc_enabled_p(); +} + +void +rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag) +{ + struct objspace *objspace = objspace_ptr; + + objspace->gc_stress = RTEST(flag); +} + +VALUE +rb_gc_impl_stress_get(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->gc_stress ? Qtrue : Qfalse; +} + +VALUE +rb_gc_impl_config_get(void *objspace_ptr) +{ + VALUE hash = rb_hash_new(); + + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_worker_count")), RB_ULONG2NUM(mmtk_worker_count())); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_plan")), rb_str_new_cstr((const char *)mmtk_plan())); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_mode")), rb_str_new_cstr((const char *)mmtk_heap_mode())); + size_t heap_min = mmtk_heap_min(); + if (heap_min > 0) rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_min")), RB_ULONG2NUM(heap_min)); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_max")), RB_ULONG2NUM(mmtk_heap_max())); + + return hash; +} + +void +rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) +{ + // TODO +} + +struct rb_gc_vm_context * +rb_gc_impl_get_vm_context(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return &objspace->vm_context; +} + +// Object allocation + +static VALUE +rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size, size_t align) +{ + MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer; + if (bump_pointer == NULL) return 0; + + uintptr_t cursor = bump_pointer->cursor; + + // Ensure cursor is aligned + size_t mask = align - 1; + cursor = (cursor + mask) & ~mask; + + cursor += size; + + if (cursor > bump_pointer->limit) { + return 0; + } + else { + VALUE obj = cursor - size; + bump_pointer->cursor = cursor; + return obj; + } +} + +static bool +obj_can_parallel_free_p(VALUE obj) +{ + switch (RB_BUILTIN_TYPE(obj)) { + case T_ARRAY: + case T_BIGNUM: + case T_COMPLEX: + case T_FLOAT: + case T_HASH: + case T_OBJECT: + case T_RATIONAL: + case T_REGEXP: + case T_STRING: + case T_STRUCT: + case T_SYMBOL: + return true; + default: + return false; + } +} + +static void +mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache) +{ + if (cache->obj_free_parallel_count > 0) { + mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf, + cache->obj_free_parallel_count, true); + cache->obj_free_parallel_count = 0; + } + if (cache->obj_free_non_parallel_count > 0) { + mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf, + cache->obj_free_non_parallel_count, false); + cache->obj_free_non_parallel_count = 0; + } +} + +static inline void +mmtk_buffer_obj_free_candidate(struct MMTk_ractor_cache *cache, VALUE obj) +{ + if (obj_can_parallel_free_p(obj)) { + cache->obj_free_parallel_buf[cache->obj_free_parallel_count++] = (MMTk_ObjectReference)obj; + if (cache->obj_free_parallel_count >= OBJ_FREE_BUF_CAPACITY) { + mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf, + cache->obj_free_parallel_count, true); + cache->obj_free_parallel_count = 0; + } + } + else { + cache->obj_free_non_parallel_buf[cache->obj_free_non_parallel_count++] = (MMTk_ObjectReference)obj; + if (cache->obj_free_non_parallel_count >= OBJ_FREE_BUF_CAPACITY) { + mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf, + cache->obj_free_non_parallel_count, false); + cache->obj_free_non_parallel_count = 0; + } + } +} + +VALUE +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) +{ +#define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0 + struct objspace *objspace = objspace_ptr; + struct MMTk_ractor_cache *ractor_cache = cache_ptr; + + if (alloc_size > MMTK_MAX_OBJ_SIZE) rb_bug("too big"); + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + if (alloc_size == heap_sizes[i]) break; + if (alloc_size < heap_sizes[i]) { + alloc_size = heap_sizes[i]; + break; + } + } + + if (objspace->gc_stress) { + mmtk_handle_user_collection_request(ractor_cache, false, false); + } + + // Layout: [hidden size header (sizeof(VALUE))][payload (alloc_size)][suffix (RB_GC_OBJ_SUFFIX_SIZE)] + alloc_size += sizeof(VALUE) + RB_GC_OBJ_SUFFIX_SIZE; + + VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN); + if (!alloc_obj) { + alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT); + } + + alloc_obj++; + alloc_obj[-1] = alloc_size - sizeof(VALUE) - RB_GC_OBJ_SUFFIX_SIZE; + alloc_obj[0] = flags; + alloc_obj[1] = klass; + + // TODO: implement fast path for mmtk_post_alloc + mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size, MMTK_ALLOCATION_SEMANTICS_DEFAULT); + + // TODO: only add when object needs obj_free to be called + mmtk_buffer_obj_free_candidate(ractor_cache, (VALUE)alloc_obj); + + objspace->total_allocated_objects++; + + return (VALUE)alloc_obj; +} + +size_t +rb_gc_impl_obj_slot_size(VALUE obj) +{ + return ((VALUE *)obj)[-1]; +} + +size_t +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) +{ + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + if (size == heap_sizes[i]) return i; + if (size < heap_sizes[i]) return i; + } + + rb_bug("size too big"); +} + +bool +rb_gc_impl_size_allocatable_p(size_t size) +{ + return size <= MMTK_MAX_OBJ_SIZE; +} + +// Malloc +void * +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + // TODO: don't use system malloc + return malloc(size); +} + +void * +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + // TODO: don't use system calloc + return calloc(1, size); +} + +void * +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) +{ + // TODO: don't use system realloc + return realloc(ptr, new_size); +} + +void +rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) +{ + // TODO: don't use system free + free(ptr); +} + +void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { } + +// Marking +static inline VALUE +rb_mmtk_call_object_closure(VALUE obj, bool pin) +{ + if (RB_UNLIKELY(RB_BUILTIN_TYPE(obj) == T_NONE)) { + enum { info_size = 256 }; + char obj_info_buf[info_size]; + rb_raw_obj_info(obj_info_buf, info_size, obj); + + char parent_obj_info_buf[info_size]; + rb_raw_obj_info(parent_obj_info_buf, info_size, marking_parent_object); + + rb_mmtk_gc_thread_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf); + } + + return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function( + rb_mmtk_gc_thread_tls->object_closure.rust_closure, + rb_mmtk_gc_thread_tls->gc_context, + (MMTk_ObjectReference)obj, + pin + ); +} + +void +rb_gc_impl_mark(void *objspace_ptr, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_call_object_closure(obj, false); +} + +void +rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) +{ + if (RB_SPECIAL_CONST_P(*ptr)) return; + + VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false); + if (new_obj != *ptr) { + *ptr = new_obj; + } +} + +void +rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_call_object_closure(obj, true); +} + +void +rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) +{ + if (rb_gc_impl_pointer_to_heap_p(objspace_ptr, (const void *)obj)) { + rb_gc_impl_mark_and_pin(objspace_ptr, obj); + } +} + +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) +{ + RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE); + mmtk_declare_weak_references((MMTk_ObjectReference)obj); +} + +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + return mmtk_weak_references_alive_p((MMTk_ObjectReference)obj); +} + +// Compaction +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + mmtk_register_pinning_obj((MMTk_ObjectReference)obj); +} + +bool +rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) +{ + return rb_mmtk_call_object_closure(obj, false) != obj; +} + +VALUE +rb_gc_impl_location(void *objspace_ptr, VALUE obj) +{ + return rb_mmtk_call_object_closure(obj, false); +} + +// Write barriers +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache(); + + if (SPECIAL_CONST_P(b)) return; + +#ifdef MMTK_DEBUG + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)a)) { + char buff[256]; + rb_bug("a: %s is not an object", rb_raw_obj_info(buff, 256, a)); + } + + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)b)) { + char buff[256]; + rb_bug("b: %s is not an object", rb_raw_obj_info(buff, 256, b)); + } +#endif + + MMTK_ASSERT(BUILTIN_TYPE(a) != T_NONE); + MMTK_ASSERT(BUILTIN_TYPE(b) != T_NONE); + + mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)a); +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj); +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache(); + + mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)obj); +} + +// Heap walking +static void +each_objects_i(MMTk_ObjectReference obj, void *d) +{ + rb_darray(VALUE) *objs = d; + + rb_darray_append(objs, (VALUE)obj); +} + +static void +each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data) +{ + rb_darray(VALUE) objs; + rb_darray_make(&objs, 0); + + mmtk_enumerate_objects(each_objects_i, &objs); + + VALUE *obj_ptr; + rb_darray_foreach(objs, i, obj_ptr) { + if (!mmtk_is_mmtk_object((MMTk_ObjectReference)*obj_ptr)) continue; + + if (func(*obj_ptr, data) != 0) { + break; + } + } + + rb_darray_free(objs); +} + +struct rb_gc_impl_each_objects_data { + int (*func)(void *, void *, size_t, void *); + void *data; +}; + +static int +rb_gc_impl_each_objects_i(VALUE obj, void *d) +{ + struct rb_gc_impl_each_objects_data *data = d; + + size_t slot_size = rb_gc_impl_obj_slot_size(obj); + + return data->func((void *)obj, (void *)(obj + slot_size), slot_size, data->data); +} + +void +rb_gc_impl_each_objects(void *objspace_ptr, int (*func)(void *, void *, size_t, void *), void *data) +{ + struct rb_gc_impl_each_objects_data each_objects_data = { + .func = func, + .data = data + }; + + each_object(objspace_ptr, rb_gc_impl_each_objects_i, &each_objects_data); +} + +struct rb_gc_impl_each_object_data { + void (*func)(VALUE, void *); + void *data; +}; + +static int +rb_gc_impl_each_object_i(VALUE obj, void *d) +{ + struct rb_gc_impl_each_object_data *data = d; + + data->func(obj, data->data); + + return 0; +} + +void +rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE, void *), void *data) +{ + struct rb_gc_impl_each_object_data each_object_data = { + .func = func, + .data = data + }; + + each_object(objspace_ptr, rb_gc_impl_each_object_i, &each_object_data); +} + +// Finalizers +static VALUE +gc_run_finalizers_get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i + 1); +} + +static void +gc_run_finalizers(void *data) +{ + struct objspace *objspace = data; + + rb_gc_set_pending_interrupt(); + + while (objspace->finalizer_jobs != NULL) { + struct MMTk_final_job *job = objspace->finalizer_jobs; + objspace->finalizer_jobs = job->next; + + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + job->as.dfree.func(job->as.dfree.data); + break; + case MMTK_FINAL_JOB_FINALIZE: { + VALUE finalizer_array = job->as.finalize.finalizer_array; + + rb_gc_run_obj_finalizer( + RARRAY_AREF(finalizer_array, 0), + RARRAY_LEN(finalizer_array) - 1, + gc_run_finalizers_get_final, + (void *)finalizer_array + ); + + RB_GC_GUARD(finalizer_array); + break; + } + } + + xfree(job); + } + + rb_gc_unset_pending_interrupt(); +} + +void +rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) +{ + if (dfree == NULL) return; + + struct objspace *objspace = objspace_ptr; + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->kind = MMTK_FINAL_JOB_DFREE; + job->as.dfree.func = dfree; + job->as.dfree.data = data; + + struct MMTk_final_job *prev; + do { + job->next = objspace->finalizer_jobs; + prev = RUBY_ATOMIC_PTR_CAS(objspace->finalizer_jobs, job->next, job); + } while (prev != job->next); + + if (!ruby_free_at_exit_p()) { + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } +} + +VALUE +rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) +{ + struct objspace *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + RBASIC(obj)->flags |= FL_FINALIZE; + + int lev = RB_GC_VM_LOCK(); + + if (st_lookup(objspace->finalizer_table, obj, &data)) { + table = (VALUE)data; + + /* avoid duplicate block, table is usually small */ + { + long len = RARRAY_LEN(table); + long i; + + for (i = 0; i < len; i++) { + VALUE recv = RARRAY_AREF(table, i); + if (rb_equal(recv, block)) { + RB_GC_VM_UNLOCK(lev); + return recv; + } + } + } + + rb_ary_push(table, block); + } + else { + table = rb_ary_new3(2, rb_obj_id(obj), block); + rb_obj_hide(table); + st_add_direct(objspace->finalizer_table, obj, table); + } + + RB_GC_VM_UNLOCK(lev); + + return block; +} + +void +rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) +{ + struct objspace *objspace = objspace_ptr; + + st_data_t data = obj; + + int lev = RB_GC_VM_LOCK(); + st_delete(objspace->finalizer_table, &data, 0); + RB_GC_VM_UNLOCK(lev); + + FL_UNSET(obj, FL_FINALIZE); +} + +void +rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) +{ + struct objspace *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + if (!FL_TEST(obj, FL_FINALIZE)) return; + + int lev = RB_GC_VM_LOCK(); + if (RB_LIKELY(st_lookup(objspace->finalizer_table, obj, &data))) { + table = rb_ary_dup((VALUE)data); + RARRAY_ASET(table, 0, rb_obj_id(dest)); + st_insert(objspace->finalizer_table, dest, table); + FL_SET(dest, FL_FINALIZE); + } + else { + rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj)); + } + RB_GC_VM_UNLOCK(lev); +} + +static int +move_finalizer_from_table_i(st_data_t key, st_data_t val, st_data_t arg) +{ + struct objspace *objspace = (struct objspace *)arg; + + make_final_job(objspace, (VALUE)key, (VALUE)val); + + return ST_DELETE; +} + +void +rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + while (objspace->finalizer_table->num_entries) { + st_foreach(objspace->finalizer_table, move_finalizer_from_table_i, (st_data_t)objspace); + + gc_run_finalizers(objspace); + } + + unsigned int lev = RB_GC_VM_LOCK(); + { + struct MMTk_ractor_cache *rc; + ccan_list_for_each(&objspace->ractor_caches, rc, list_node) { + mmtk_flush_obj_free_buffer(rc); + } + + struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates(); + for (size_t i = 0; i < registered_candidates.len; i++) { + VALUE obj = (VALUE)registered_candidates.ptr[i]; + + if (rb_gc_shutdown_call_finalizer_p(obj)) { + rb_gc_obj_free(objspace_ptr, obj); + RBASIC(obj)->flags = 0; + } + } + mmtk_free_raw_vec_of_obj_ref(registered_candidates); + } + RB_GC_VM_UNLOCK(lev); + + gc_run_finalizers(objspace); +} + +// Forking + +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + retry: + objspace->fork_hook_vm_lock_lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + /* At this point, we know that all the Ractors are paused because of the + * rb_gc_vm_barrier above. Since rb_mmtk_block_for_gc is a barrier point, + * one or more Ractors could be paused there. However, mmtk_before_fork is + * not compatible with that because it assumes that the MMTk workers are idle, + * but the workers are not idle because they are busy working on a GC. + * + * This essentially implements a trylock. It will optimistically lock but will + * release the lock if it detects that any other Ractors are waiting in + * rb_mmtk_block_for_gc. + */ + rb_atomic_t mutator_blocking_count = RUBY_ATOMIC_LOAD(objspace->mutator_blocking_count); + if (mutator_blocking_count != 0) { + RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev); + goto retry; + } + + mmtk_before_fork(); +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + struct objspace *objspace = objspace_ptr; + + mmtk_after_fork(rb_gc_get_ractor_newobj_cache()); + + RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev); +} + +// Statistics + +void +rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) +{ + struct objspace *objspace = objspace_ptr; + + objspace->measure_gc_time = RTEST(flag); +} + +bool +rb_gc_impl_get_measure_total_time(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->measure_gc_time; +} + +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->total_gc_time; +} + +size_t +rb_gc_impl_gc_count(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->gc_count; +} + +VALUE +rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE hash_or_key) +{ + VALUE hash = Qnil, key = Qnil; + + if (SYMBOL_P(hash_or_key)) { + key = hash_or_key; + } + else if (RB_TYPE_P(hash_or_key, T_HASH)) { + hash = hash_or_key; + } + else { + rb_bug("gc_info_decode: non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == ID2SYM(rb_intern_const(#name))) \ + return (attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, ID2SYM(rb_intern_const(#name)), (attr)); + + /* Hack to get StackProf working because it calls rb_gc_latest_gc_info with + * the :state key and expects a result. This always returns the :none state. */ + SET(state, ID2SYM(rb_intern_const("none"))); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +enum gc_stat_sym { + gc_stat_sym_count, + gc_stat_sym_moving_gc_count, + gc_stat_sym_time, + gc_stat_sym_total_allocated_objects, + gc_stat_sym_total_bytes, + gc_stat_sym_used_bytes, + gc_stat_sym_free_bytes, + gc_stat_sym_starting_heap_address, + gc_stat_sym_last_heap_address, + gc_stat_sym_weak_references_count, + gc_stat_sym_last +}; + +static VALUE gc_stat_symbols[gc_stat_sym_last]; + +static void +setup_gc_stat_symbols(void) +{ + if (gc_stat_symbols[0] == 0) { +#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s)) + S(count); + S(moving_gc_count); + S(time); + S(total_allocated_objects); + S(total_bytes); + S(used_bytes); + S(free_bytes); + S(starting_heap_address); + S(last_heap_address); + S(weak_references_count); + } +} + +VALUE +rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) +{ + struct objspace *objspace = objspace_ptr; + VALUE hash = Qnil, key = Qnil; + + setup_gc_stat_symbols(); + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + hash = hash_or_sym; + } + else if (SYMBOL_P(hash_or_sym)) { + key = hash_or_sym; + } + else { + rb_bug("non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == gc_stat_symbols[gc_stat_sym_##name]) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr)); + + SET(count, objspace->gc_count); + SET(moving_gc_count, objspace->moving_gc_count); + SET(time, objspace->total_gc_time / (1000 * 1000)); + SET(total_allocated_objects, objspace->total_allocated_objects); + SET(total_bytes, mmtk_total_bytes()); + SET(used_bytes, mmtk_used_bytes()); + SET(free_bytes, mmtk_free_bytes()); + SET(starting_heap_address, (size_t)mmtk_starting_heap_address()); + SET(last_heap_address, (size_t)mmtk_last_heap_address()); + SET(weak_references_count, mmtk_weak_references_count()); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +VALUE +rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) +{ + if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) { + int heap_idx = FIX2INT(heap_name); + if (heap_idx < 0 || heap_idx >= MMTK_HEAP_COUNT) { + rb_raise(rb_eArgError, "size pool index out of range"); + } + + if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) { + return SIZET2NUM(heap_sizes[heap_idx]); + } + + return Qundef; + } + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + return hash_or_sym; + } + + return Qundef; +} + +// Miscellaneous + +#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 1 +static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) +{ + static ID ID_object_id; + + if (!ID_object_id) { +#define I(s) ID_##s = rb_intern(#s); + I(object_id); +#undef I + } + + size_t n = 0; + +#define SET_ENTRY(na, v) do { \ + MMTK_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; + + return object_metadata_entries; +} + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + if (ptr == NULL) return false; + if ((uintptr_t)ptr % sizeof(void*) != 0) return false; + return mmtk_is_mmtk_object((MMTk_Address)ptr); +} + +bool +rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj) +{ + return false; +} + +void rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) { } + +void +rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) +{ + if (mmtk_object_wb_unprotected_p((MMTk_ObjectReference)obj)) { + rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest); + } + + rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj); +} + +// GC Identification + +const char * +rb_gc_impl_active_gc_name(void) +{ + return "mmtk"; +} diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h new file mode 100644 index 0000000000..b11e2873e3 --- /dev/null +++ b/gc/mmtk/mmtk.h @@ -0,0 +1,175 @@ +#ifndef MMTK_H +#define MMTK_H + +/* Warning, this file is autogenerated by cbindgen from the mmtk-ruby repository. Don't modify this manually. */ + +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +typedef struct MMTk_Builder MMTk_Builder; +typedef struct MMTk_Mutator MMTk_Mutator; + +typedef struct MMTk_ractor_cache *MMTk_VMThread; +typedef struct MMTk_ractor_cache *MMTk_VMMutatorThread; +typedef struct MMTk_GCThreadTLS *MMTk_VMWorkerThread; +typedef void *MMTk_Address; +typedef void *MMTk_ObjectReference; +typedef void *MMTk_NullableObjectReference; +typedef uint32_t MMTk_AllocationSemantics; + +typedef struct MMTk_BumpPointer { + uintptr_t cursor; + uintptr_t limit; +} MMTk_BumpPointer; + + +#define MMTk_OBJREF_OFFSET 8 + +#define MMTk_MIN_OBJ_ALIGN 8 + +#define MMTk_GC_THREAD_KIND_WORKER 1 + +typedef struct MMTk_RubyBindingOptions { + size_t suffix_size; +} MMTk_RubyBindingOptions; + +typedef MMTk_ObjectReference (*MMTk_ObjectClosureFunction)(void*, void*, MMTk_ObjectReference, bool); + +typedef struct MMTk_ObjectClosure { + /** + * The function to be called from C. + */ + MMTk_ObjectClosureFunction c_function; + /** + * The pointer to the Rust-level closure object. + */ + void *rust_closure; +} MMTk_ObjectClosure; + +typedef struct MMTk_GCThreadTLS { + int kind; + void *gc_context; + struct MMTk_ObjectClosure object_closure; +} MMTk_GCThreadTLS; + +typedef struct MMTk_RubyUpcalls { + void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls); + bool (*is_mutator)(void); + void (*stop_the_world)(void); + void (*resume_mutators)(bool gc_may_move); + void (*block_for_gc)(MMTk_VMMutatorThread tls); + void (*before_updating_jit_code)(void); + void (*after_updating_jit_code)(void); + size_t (*number_of_mutators)(void); + void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data); + void (*scan_gc_roots)(void); + void (*scan_objspace)(void); + void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to); + void (*update_object_references)(MMTk_ObjectReference object); + void (*call_gc_mark_children)(MMTk_ObjectReference object); + void (*handle_weak_references)(MMTk_ObjectReference object, bool moving); + void (*call_obj_free)(MMTk_ObjectReference object); + size_t (*vm_live_bytes)(void); + void (*update_global_tables)(int tbl_idx, bool moving); + int (*global_tables_count)(void); + void (*update_finalizer_table)(void); + bool (*special_const_p)(MMTk_ObjectReference object); + void (*mutator_thread_panic_handler)(void); + void (*gc_thread_panic_handler)(void); +} MMTk_RubyUpcalls; + +typedef struct MMTk_RawVecOfObjRef { + MMTk_ObjectReference *ptr; + size_t len; + size_t capa; +} MMTk_RawVecOfObjRef; + +bool mmtk_is_live_object(MMTk_ObjectReference object); + +bool mmtk_is_reachable(MMTk_ObjectReference object); + +MMTk_Builder *mmtk_builder_default(void); + +void mmtk_init_binding(MMTk_Builder *builder, + const struct MMTk_RubyBindingOptions *binding_options, + const struct MMTk_RubyUpcalls *upcalls); + +void mmtk_initialize_collection(MMTk_VMThread tls); + +MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls); + +MMTk_BumpPointer *mmtk_get_bump_pointer_allocator(MMTk_Mutator *m); + +void mmtk_destroy_mutator(MMTk_Mutator *mutator); + +void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls, bool force, bool exhaustive); + +void mmtk_set_gc_enabled(bool enable); + +bool mmtk_gc_enabled_p(void); + +MMTk_Address mmtk_alloc(MMTk_Mutator *mutator, + size_t size, + size_t align, + size_t offset, + MMTk_AllocationSemantics semantics); + +void mmtk_post_alloc(MMTk_Mutator *mutator, + MMTk_ObjectReference refer, + size_t bytes, + MMTk_AllocationSemantics semantics); + +void mmtk_add_obj_free_candidates(const MMTk_ObjectReference *objects, + size_t count, + bool can_parallel_free); + +void mmtk_declare_weak_references(MMTk_ObjectReference object); + +bool mmtk_weak_references_alive_p(MMTk_ObjectReference object); + +size_t mmtk_weak_references_count(void); + +void mmtk_register_pinning_obj(MMTk_ObjectReference obj); + +void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object); + +void mmtk_register_wb_unprotected_object(MMTk_ObjectReference object); + +bool mmtk_object_wb_unprotected_p(MMTk_ObjectReference object); + +void mmtk_enumerate_objects(void (*callback)(MMTk_ObjectReference, void*), void *data); + +struct MMTk_RawVecOfObjRef mmtk_get_all_obj_free_candidates(void); + +void mmtk_free_raw_vec_of_obj_ref(struct MMTk_RawVecOfObjRef raw_vec); + +void mmtk_before_fork(void); + +void mmtk_after_fork(MMTk_VMThread tls); + +size_t mmtk_total_bytes(void); + +size_t mmtk_used_bytes(void); + +size_t mmtk_free_bytes(void); + +MMTk_Address mmtk_starting_heap_address(void); + +MMTk_Address mmtk_last_heap_address(void); + +size_t mmtk_worker_count(void); + +const uint8_t *mmtk_plan(void); + +const uint8_t *mmtk_heap_mode(void); + +size_t mmtk_heap_min(void); + +size_t mmtk_heap_max(void); + +bool mmtk_is_mmtk_object(MMTk_Address addr); + +#endif /* MMTK_H */ diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs new file mode 100644 index 0000000000..30890e0853 --- /dev/null +++ b/gc/mmtk/src/abi.rs @@ -0,0 +1,335 @@ +use crate::api::RubyMutator; +use crate::extra_assert; +use crate::Ruby; +use libc::c_int; +use mmtk::scheduler::GCWorker; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMWorkerThread; + +// For the C binding +pub const OBJREF_OFFSET: usize = 8; +pub const MIN_OBJ_ALIGN: usize = 8; // Even on 32-bit machine. A Ruby object is at least 40 bytes large. + +pub const GC_THREAD_KIND_WORKER: libc::c_int = 1; + +const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; + +// An opaque type for the C counterpart. +#[allow(non_camel_case_types)] +pub struct st_table; + +#[repr(C)] +pub struct HiddenHeader { + pub prefix: usize, +} + +impl HiddenHeader { + #[inline(always)] + pub fn is_sane(&self) -> bool { + self.prefix & !HIDDEN_SIZE_MASK == 0 + } + + #[inline(always)] + fn assert_sane(&self) { + extra_assert!( + self.is_sane(), + "Hidden header is corrupted: {:x}", + self.prefix + ); + } + + pub fn payload_size(&self) -> usize { + self.assert_sane(); + self.prefix & HIDDEN_SIZE_MASK + } +} + +/// Provide convenient methods for accessing Ruby objects. +/// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. +pub struct RubyObjectAccess { + objref: ObjectReference, +} + +impl RubyObjectAccess { + pub fn from_objref(objref: ObjectReference) -> Self { + Self { objref } + } + + pub fn obj_start(&self) -> Address { + self.objref.to_raw_address().sub(Self::prefix_size()) + } + + pub fn payload_addr(&self) -> Address { + self.objref.to_raw_address() + } + + pub fn suffix_addr(&self) -> Address { + self.objref.to_raw_address().add(self.payload_size()) + } + + pub fn obj_end(&self) -> Address { + self.suffix_addr() + Self::suffix_size() + } + + fn hidden_header(&self) -> &'static HiddenHeader { + unsafe { self.obj_start().as_ref() } + } + + #[allow(unused)] // Maybe we need to mutate the hidden header in the future. + fn hidden_header_mut(&self) -> &'static mut HiddenHeader { + unsafe { self.obj_start().as_mut_ref() } + } + + pub fn payload_size(&self) -> usize { + self.hidden_header().payload_size() + } + + fn flags_field(&self) -> Address { + self.objref.to_raw_address() + } + + pub fn load_flags(&self) -> usize { + unsafe { self.flags_field().load::<usize>() } + } + + pub fn prefix_size() -> usize { + // Currently, a hidden size field of word size is placed before each object. + OBJREF_OFFSET + } + + pub fn suffix_size() -> usize { + // In RACTOR_CHECK_MODE, Ruby hides a field after each object to hold the Ractor ID. + unsafe { crate::BINDING_FAST.suffix_size } + } + + pub fn object_size(&self) -> usize { + Self::prefix_size() + self.payload_size() + Self::suffix_size() + } +} + +type ObjectClosureFunction = + extern "C" fn(*mut libc::c_void, *mut libc::c_void, ObjectReference, bool) -> ObjectReference; + +#[repr(C)] +pub struct ObjectClosure { + /// The function to be called from C. + pub c_function: ObjectClosureFunction, + /// The pointer to the Rust-level closure object. + pub rust_closure: *mut libc::c_void, +} + +impl Default for ObjectClosure { + fn default() -> Self { + Self { + c_function: THE_UNREGISTERED_CLOSURE_FUNC, + rust_closure: std::ptr::null_mut(), + } + } +} + +/// Rust doesn't require function items to have a unique address. +/// We therefore force using this particular constant. +/// +/// See: https://rust-lang.github.io/rust-clippy/master/index.html#fn_address_comparisons +const THE_UNREGISTERED_CLOSURE_FUNC: ObjectClosureFunction = ObjectClosure::c_function_unregistered; + +impl ObjectClosure { + /// Set this ObjectClosure temporarily to `visit_object`, and execute `f`. During the execution of + /// `f`, the Ruby VM may call this ObjectClosure. When the Ruby VM calls this ObjectClosure, + /// it effectively calls `visit_object`. + /// + /// This method is intended to run Ruby VM code in `f` with temporarily modified behavior of + /// `rb_gc_mark`, `rb_gc_mark_movable` and `rb_gc_location` + /// + /// Both `f` and `visit_object` may access and modify local variables in the environment where + /// `set_temporarily_and_run_code` called. + /// + /// Note that this function is not reentrant. Don't call this function in either `callback` or + /// `f`. + pub fn set_temporarily_and_run_code<'env, T, F1, F2>( + &mut self, + mut visit_object: F1, + f: F2, + ) -> T + where + F1: 'env + FnMut(&'static mut GCWorker<Ruby>, ObjectReference, bool) -> ObjectReference, + F2: 'env + FnOnce() -> T, + { + debug_assert!( + std::ptr::fn_addr_eq(self.c_function, THE_UNREGISTERED_CLOSURE_FUNC), + "set_temporarily_and_run_code is recursively called." + ); + self.c_function = Self::c_function_registered::<F1>; + self.rust_closure = &mut visit_object as *mut F1 as *mut libc::c_void; + let result = f(); + *self = Default::default(); + result + } + + extern "C" fn c_function_registered<F>( + rust_closure: *mut libc::c_void, + worker: *mut libc::c_void, + object: ObjectReference, + pin: bool, + ) -> ObjectReference + where + F: FnMut(&'static mut GCWorker<Ruby>, ObjectReference, bool) -> ObjectReference, + { + let rust_closure = unsafe { &mut *(rust_closure as *mut F) }; + let worker = unsafe { &mut *(worker as *mut GCWorker<Ruby>) }; + rust_closure(worker, object, pin) + } + + extern "C" fn c_function_unregistered( + _rust_closure: *mut libc::c_void, + worker: *mut libc::c_void, + object: ObjectReference, + pin: bool, + ) -> ObjectReference { + let worker = unsafe { &mut *(worker as *mut GCWorker<Ruby>) }; + panic!( + "object_closure is not set. worker ordinal: {}, object: {}, pin: {}", + worker.ordinal, object, pin + ); + } +} + +#[repr(C)] +pub struct GCThreadTLS { + pub kind: libc::c_int, + pub gc_context: *mut libc::c_void, + pub object_closure: ObjectClosure, +} + +impl GCThreadTLS { + fn new(kind: libc::c_int, gc_context: *mut libc::c_void) -> Self { + Self { + kind, + gc_context, + object_closure: Default::default(), + } + } + + pub fn for_worker(gc_context: *mut GCWorker<Ruby>) -> Self { + Self::new(GC_THREAD_KIND_WORKER, gc_context as *mut libc::c_void) + } + + pub fn from_vwt(vwt: VMWorkerThread) -> *mut GCThreadTLS { + unsafe { std::mem::transmute(vwt) } + } + + /// Cast a pointer to `GCThreadTLS` to a ref, with assertion for null pointer. + /// + /// # Safety + /// + /// Has undefined behavior if `ptr` is invalid. + pub unsafe fn check_cast(ptr: *mut GCThreadTLS) -> &'static mut GCThreadTLS { + assert!(!ptr.is_null()); + let result = unsafe { &mut *ptr }; + debug_assert!({ + let kind = result.kind; + kind == GC_THREAD_KIND_WORKER + }); + result + } + + /// Cast a pointer to `VMWorkerThread` to a ref, with assertion for null pointer. + /// + /// # Safety + /// + /// Has undefined behavior if `ptr` is invalid. + pub unsafe fn from_vwt_check(vwt: VMWorkerThread) -> &'static mut GCThreadTLS { + let ptr = Self::from_vwt(vwt); + unsafe { Self::check_cast(ptr) } + } + + #[allow(clippy::not_unsafe_ptr_arg_deref)] // `transmute` does not dereference pointer + pub fn to_vwt(ptr: *mut Self) -> VMWorkerThread { + unsafe { std::mem::transmute(ptr) } + } + + pub fn worker<'w>(&mut self) -> &'w mut GCWorker<Ruby> { + // NOTE: The returned ref points to the worker which does not have the same lifetime as self. + assert!(self.kind == GC_THREAD_KIND_WORKER); + unsafe { &mut *(self.gc_context as *mut GCWorker<Ruby>) } + } +} + +#[repr(C)] +#[derive(Clone)] +pub struct RawVecOfObjRef { + pub ptr: *mut ObjectReference, + pub len: usize, + pub capa: usize, +} + +impl RawVecOfObjRef { + pub fn from_vec(vec: Vec<ObjectReference>) -> RawVecOfObjRef { + // Note: Vec::into_raw_parts is unstable. We implement it manually. + let mut vec = std::mem::ManuallyDrop::new(vec); + let (ptr, len, capa) = (vec.as_mut_ptr(), vec.len(), vec.capacity()); + + RawVecOfObjRef { ptr, len, capa } + } + + /// # Safety + /// + /// This function turns raw pointer into a Vec without check. + pub unsafe fn into_vec(self) -> Vec<ObjectReference> { + unsafe { Vec::from_raw_parts(self.ptr, self.len, self.capa) } + } +} + +impl From<Vec<ObjectReference>> for RawVecOfObjRef { + fn from(v: Vec<ObjectReference>) -> Self { + Self::from_vec(v) + } +} + +#[repr(C)] +#[derive(Clone)] +pub struct RubyBindingOptions { + pub suffix_size: usize, +} + +#[repr(C)] +#[derive(Clone)] +pub struct RubyUpcalls { + pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS), + pub is_mutator: extern "C" fn() -> bool, + pub stop_the_world: extern "C" fn(), + pub resume_mutators: extern "C" fn(gc_may_move: bool), + pub block_for_gc: extern "C" fn(tls: VMMutatorThread), + pub before_updating_jit_code: extern "C" fn(), + pub after_updating_jit_code: extern "C" fn(), + pub number_of_mutators: extern "C" fn() -> usize, + pub get_mutators: extern "C" fn( + visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void), + data: *mut libc::c_void, + ), + pub scan_gc_roots: extern "C" fn(), + pub scan_objspace: extern "C" fn(), + pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference), + pub update_object_references: extern "C" fn(object: ObjectReference), + pub call_gc_mark_children: extern "C" fn(object: ObjectReference), + pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool), + pub call_obj_free: extern "C" fn(object: ObjectReference), + pub vm_live_bytes: extern "C" fn() -> usize, + pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool), + pub global_tables_count: extern "C" fn() -> c_int, + pub update_finalizer_table: extern "C" fn(), + pub special_const_p: extern "C" fn(object: ObjectReference) -> bool, + pub mutator_thread_panic_handler: extern "C" fn(), + pub gc_thread_panic_handler: extern "C" fn(), +} + +unsafe impl Sync for RubyUpcalls {} + +#[repr(C)] +#[derive(Clone)] +pub struct HeapBounds { + pub start: *mut libc::c_void, + pub end: *mut libc::c_void, +} diff --git a/gc/mmtk/src/active_plan.rs b/gc/mmtk/src/active_plan.rs new file mode 100644 index 0000000000..80372a7576 --- /dev/null +++ b/gc/mmtk/src/active_plan.rs @@ -0,0 +1,56 @@ +use std::collections::VecDeque; +use std::marker::PhantomData; + +use crate::mmtk; +use crate::upcalls; +use crate::Ruby; +use mmtk::util::opaque_pointer::*; +use mmtk::vm::ActivePlan; +use mmtk::Mutator; + +pub struct VMActivePlan {} + +impl ActivePlan<Ruby> for VMActivePlan { + fn number_of_mutators() -> usize { + (upcalls().number_of_mutators)() + } + + fn is_mutator(_tls: VMThread) -> bool { + (upcalls().is_mutator)() + } + + fn mutator(_tls: VMMutatorThread) -> &'static mut Mutator<Ruby> { + unimplemented!() + } + + fn mutators<'a>() -> Box<dyn Iterator<Item = &'a mut Mutator<Ruby>> + 'a> { + let mut mutators = VecDeque::new(); + (upcalls().get_mutators)( + add_mutator_to_vec, + &mut mutators as *mut VecDeque<&mut Mutator<Ruby>> as _, + ); + + Box::new(RubyMutatorIterator { + mutators, + phantom_data: PhantomData, + }) + } +} + +extern "C" fn add_mutator_to_vec(mutator: *mut Mutator<Ruby>, mutators: *mut libc::c_void) { + let mutators = unsafe { &mut *(mutators as *mut VecDeque<*mut Mutator<Ruby>>) }; + mutators.push_back(unsafe { &mut *mutator }); +} + +struct RubyMutatorIterator<'a> { + mutators: VecDeque<&'a mut Mutator<Ruby>>, + phantom_data: PhantomData<&'a ()>, +} + +impl<'a> Iterator for RubyMutatorIterator<'a> { + type Item = &'a mut Mutator<Ruby>; + + fn next(&mut self) -> Option<Self::Item> { + self.mutators.pop_front() + } +} diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs new file mode 100644 index 0000000000..c0540fe0c8 --- /dev/null +++ b/gc/mmtk/src/api.rs @@ -0,0 +1,551 @@ +// Functions in this module are unsafe for one reason: +// They are called by C functions and they need to pass raw pointers to Rust. +#![allow(clippy::missing_safety_doc)] + +use mmtk::util::alloc::BumpPointer; +use mmtk::util::alloc::ImmixAllocator; +use mmtk::util::conversions; +use mmtk::util::options::PlanSelector; +use std::str::FromStr; +use std::sync::atomic::Ordering; + +use crate::abi::RawVecOfObjRef; +use crate::abi::RubyBindingOptions; +use crate::abi::RubyUpcalls; +use crate::binding; +use crate::binding::RubyBinding; +use crate::heap::CpuHeapTriggerConfig; +use crate::heap::RubyHeapTriggerConfig; +use crate::heap::CPU_HEAP_TRIGGER_CONFIG; +use crate::heap::RUBY_HEAP_TRIGGER_CONFIG; +use crate::mmtk; +use crate::utils::default_heap_max; +use crate::utils::parse_capacity; +use crate::Ruby; +use crate::RubySlot; +use mmtk::memory_manager; +use mmtk::memory_manager::mmtk_init; +use mmtk::util::constants::MIN_OBJECT_SIZE; +use mmtk::util::options::GCTriggerSelector; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; +use mmtk::AllocationSemantics; +use mmtk::MMTKBuilder; +use mmtk::Mutator; + +pub type RubyMutator = Mutator<Ruby>; + +#[no_mangle] +pub extern "C" fn mmtk_is_live_object(object: ObjectReference) -> bool { + memory_manager::is_live_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_is_reachable(object: ObjectReference) -> bool { + object.is_reachable() +} + +// =============== Bootup =============== + +fn parse_env_var_with<T, F: FnOnce(&str) -> Option<T>>(key: &str, parse: F) -> Option<T> { + let val = match std::env::var(key) { + Ok(val) => val, + Err(std::env::VarError::NotPresent) => return None, + Err(std::env::VarError::NotUnicode(os_string)) => { + eprintln!("[FATAL] Invalid {key} {os_string:?}"); + std::process::exit(1); + } + }; + + let parsed = parse(&val).unwrap_or_else(|| { + eprintln!("[FATAL] Invalid {key} {val}"); + std::process::exit(1); + }); + + Some(parsed) +} + +fn parse_env_var<T: FromStr>(key: &str) -> Option<T> { + parse_env_var_with(key, |s| s.parse().ok()) +} + +fn mmtk_builder_default_parse_threads() -> Option<usize> { + parse_env_var("MMTK_THREADS") +} + +fn mmtk_builder_default_parse_heap_min() -> usize { + const DEFAULT_HEAP_MIN: usize = 1 << 20; + parse_env_var_with("MMTK_HEAP_MIN", parse_capacity).unwrap_or(DEFAULT_HEAP_MIN) +} + +fn mmtk_builder_default_parse_heap_max() -> usize { + parse_env_var_with("MMTK_HEAP_MAX", parse_capacity).unwrap_or_else(default_heap_max) +} + +fn parse_float_env_var(key: &str, default: f64, min: f64, max: f64) -> f64 { + parse_env_var_with(key, |s| { + let mut float = f64::from_str(s).unwrap_or(default); + + if float <= min { + eprintln!( + "{key} has value {float} which must be greater than {min}, using default instead" + ); + float = default; + } + + if float >= max { + eprintln!( + "{key} has value {float} which must be less than {max}, using default instead" + ); + float = default; + } + + Some(float) + }) + .unwrap_or(default) +} + +fn mmtk_builder_default_parse_heap_mode(heap_min: usize, heap_max: usize) -> GCTriggerSelector { + let make_fixed = || GCTriggerSelector::FixedHeapSize(heap_max); + let make_dynamic = || GCTriggerSelector::DynamicHeapSize(heap_min, heap_max); + + parse_env_var_with("MMTK_HEAP_MODE", |s| match s { + "fixed" => Some(make_fixed()), + "dynamic" => Some(make_dynamic()), + "ruby" => { + let min_ratio = parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", 0.2, 0.0, 1.0); + let goal_ratio = + parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", 0.4, min_ratio, 1.0); + let max_ratio = + parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", 0.65, goal_ratio, 1.0); + + crate::heap::RUBY_HEAP_TRIGGER_CONFIG + .set(RubyHeapTriggerConfig { + min_heap_pages: conversions::bytes_to_pages_up(heap_min), + max_heap_pages: conversions::bytes_to_pages_up(heap_max), + heap_pages_min_ratio: min_ratio, + heap_pages_goal_ratio: goal_ratio, + heap_pages_max_ratio: max_ratio, + }) + .unwrap_or_else(|_| panic!("RUBY_HEAP_TRIGGER_CONFIG is already set")); + + Some(GCTriggerSelector::Delegated) + } + "cpu" => { + // CPU-overhead-driven heap sizing based on Tavakolisomeh et al., + // "Heap Size Adjustment with CPU Control", MPLR '23. + // + // Target is expressed as a percentage (0, 100) via + // `MMTK_GC_CPU_TARGET`. The paper recommends 15 for ZGC (a + // concurrent collector); we default to 5 for MMTk-Ruby. With + // MMTk's stop-the-world Immix, every percent of GC CPU is also + // a percent of wall-clock the mutator is blocked on, so a much + // smaller budget is appropriate. An empirical sweep across + // ruby-bench (railsbench, lobsters, psych-load, liquid-render, + // lee) found target=5 to be Pareto-optimal: ~6% geomean speedup + // vs. the `ruby` heap mode with effectively identical geomean + // peak RSS. + let target_percent = parse_float_env_var("MMTK_GC_CPU_TARGET", 5.0, 0.0, 100.0); + let window_size = parse_env_var::<usize>("MMTK_GC_CPU_WINDOW").unwrap_or(3); + let window_size = window_size.max(1); + + let min_heap_pages = conversions::bytes_to_pages_up(heap_min); + let max_heap_pages = conversions::bytes_to_pages_up(heap_max); + // Start at the min heap size, as the other delegated triggers do. + // The control loop will adjust from here after the first GC cycle. + let initial_heap_pages = min_heap_pages; + + CPU_HEAP_TRIGGER_CONFIG + .set(CpuHeapTriggerConfig { + min_heap_pages, + max_heap_pages, + initial_heap_pages, + target_gc_cpu: target_percent / 100.0, + window_size, + }) + .unwrap_or_else(|_| panic!("CPU_HEAP_TRIGGER_CONFIG is already set")); + + Some(GCTriggerSelector::Delegated) + } + _ => None, + }) + .unwrap_or_else(make_dynamic) +} + +fn mmtk_builder_default_parse_plan() -> PlanSelector { + parse_env_var_with("MMTK_PLAN", |s| match s { + "NoGC" => Some(PlanSelector::NoGC), + "MarkSweep" => Some(PlanSelector::MarkSweep), + "Immix" => Some(PlanSelector::Immix), + _ => None, + }) + .unwrap_or(PlanSelector::Immix) +} + +#[no_mangle] +pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder { + let mut builder = MMTKBuilder::new_no_env_vars(); + builder.options.no_finalizer.set(true); + + if let Some(threads) = mmtk_builder_default_parse_threads() { + if !builder.options.threads.set(threads) { + // MMTk will validate it and reject 0. + eprintln!("[FATAL] Failed to set the number of MMTk threads to {threads}"); + std::process::exit(1); + } + } + + let heap_min = mmtk_builder_default_parse_heap_min(); + + let heap_max = mmtk_builder_default_parse_heap_max(); + + if heap_min >= heap_max { + eprintln!("[FATAL] MMTK_HEAP_MIN({heap_min}) >= MMTK_HEAP_MAX({heap_max})"); + std::process::exit(1); + } + + builder + .options + .gc_trigger + .set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max)); + + builder.options.plan.set(mmtk_builder_default_parse_plan()); + + Box::into_raw(Box::new(builder)) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_init_binding( + builder: *mut MMTKBuilder, + binding_options: *const RubyBindingOptions, + upcalls: *const RubyUpcalls, +) { + crate::MUTATOR_THREAD_PANIC_HANDLER + .set((unsafe { (*upcalls).clone() }).mutator_thread_panic_handler) + .unwrap_or_else(|_| panic!("MUTATOR_THREAD_PANIC_HANDLER is already initialized")); + + crate::set_panic_hook(); + + let builder: Box<MMTKBuilder> = unsafe { Box::from_raw(builder) }; + let binding_options = unsafe { (*binding_options).clone() }; + let mmtk_boxed = mmtk_init(&builder); + let mmtk_static = Box::leak(Box::new(mmtk_boxed)); + + let mut binding = RubyBinding::new(mmtk_static, &binding_options, upcalls); + binding + .weak_proc + .init_parallel_obj_free_candidates(memory_manager::num_of_workers(binding.mmtk)); + + crate::BINDING + .set(binding) + .unwrap_or_else(|_| panic!("Binding is already initialized")); +} + +#[no_mangle] +pub extern "C" fn mmtk_initialize_collection(tls: VMThread) { + memory_manager::initialize_collection(mmtk(), tls) +} + +#[no_mangle] +pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator { + Box::into_raw(memory_manager::bind_mutator(mmtk(), tls)) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_get_bump_pointer_allocator(m: *mut RubyMutator) -> *mut BumpPointer { + match *crate::BINDING.get().unwrap().mmtk.get_options().plan { + PlanSelector::Immix => { + let mutator: &mut Mutator<Ruby> = unsafe { &mut *m }; + let allocator = + unsafe { mutator.allocator_mut(mmtk::util::alloc::AllocatorSelector::Immix(0)) }; + + if let Some(immix_allocator) = allocator.downcast_mut::<ImmixAllocator<Ruby>>() { + &mut immix_allocator.bump_pointer as *mut BumpPointer + } else { + panic!("Failed to get bump pointer allocator"); + } + } + _ => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) { + // notify mmtk-core about destroyed mutator + memory_manager::destroy_mutator(unsafe { &mut *mutator }); + // turn the ptr back to a box, and let Rust properly reclaim it + let _ = unsafe { Box::from_raw(mutator) }; +} + +// =============== GC =============== + +#[no_mangle] +pub extern "C" fn mmtk_handle_user_collection_request( + tls: VMMutatorThread, + force: bool, + exhaustive: bool, +) { + crate::mmtk().handle_user_collection_request(tls, force, exhaustive); +} + +#[no_mangle] +pub extern "C" fn mmtk_set_gc_enabled(enable: bool) { + crate::CONFIGURATION + .gc_enabled + .store(enable, Ordering::Relaxed); +} + +#[no_mangle] +pub extern "C" fn mmtk_gc_enabled_p() -> bool { + crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed) +} + +// =============== Object allocation =============== + +#[no_mangle] +pub unsafe extern "C" fn mmtk_alloc( + mutator: *mut RubyMutator, + size: usize, + align: usize, + offset: usize, + semantics: AllocationSemantics, +) -> Address { + let clamped_size = size.max(MIN_OBJECT_SIZE); + memory_manager::alloc::<Ruby>( + unsafe { &mut *mutator }, + clamped_size, + align, + offset, + semantics, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_post_alloc( + mutator: *mut RubyMutator, + refer: ObjectReference, + bytes: usize, + semantics: AllocationSemantics, +) { + memory_manager::post_alloc::<Ruby>(unsafe { &mut *mutator }, refer, bytes, semantics) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_add_obj_free_candidates( + objects: *const ObjectReference, + count: usize, + can_parallel_free: bool, +) { + let objects = unsafe { std::slice::from_raw_parts(objects, count) }; + binding() + .weak_proc + .add_obj_free_candidates_batch(objects, can_parallel_free) +} + +// =============== Weak references =============== + +#[no_mangle] +pub extern "C" fn mmtk_declare_weak_references(object: ObjectReference) { + binding().weak_proc.add_weak_reference(object); +} + +#[no_mangle] +pub extern "C" fn mmtk_weak_references_alive_p(object: ObjectReference) -> bool { + object.is_reachable() +} + +#[no_mangle] +pub extern "C" fn mmtk_weak_references_count() -> usize { + binding().weak_proc.weak_references_count() +} + +// =============== Compaction =============== + +#[no_mangle] +pub extern "C" fn mmtk_register_pinning_obj(obj: ObjectReference) { + crate::binding().pinning_registry.register(obj); +} + +// =============== Write barriers =============== + +#[no_mangle] +pub unsafe extern "C" fn mmtk_object_reference_write_post( + mutator: *mut RubyMutator, + object: ObjectReference, +) { + let ignored_slot = RubySlot::from_address(Address::ZERO); + let ignored_target = ObjectReference::from_raw_address(Address::ZERO); + mmtk::memory_manager::object_reference_write_post( + unsafe { &mut *mutator }, + object, + ignored_slot, + ignored_target, + ) +} + +#[no_mangle] +pub extern "C" fn mmtk_register_wb_unprotected_object(object: ObjectReference) { + crate::binding().register_wb_unprotected_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_object_wb_unprotected_p(object: ObjectReference) -> bool { + crate::binding().object_wb_unprotected_p(object) +} + +// =============== Heap walking =============== + +#[no_mangle] +pub extern "C" fn mmtk_enumerate_objects( + callback: extern "C" fn(ObjectReference, *mut libc::c_void), + data: *mut libc::c_void, +) { + crate::mmtk().enumerate_objects(|object| { + callback(object, data); + }) +} + +// =============== Finalizers =============== + +#[no_mangle] +pub extern "C" fn mmtk_get_all_obj_free_candidates() -> RawVecOfObjRef { + let vec = binding().weak_proc.get_all_obj_free_candidates(); + RawVecOfObjRef::from_vec(vec) +} + +#[no_mangle] +pub extern "C" fn mmtk_free_raw_vec_of_obj_ref(raw_vec: RawVecOfObjRef) { + unsafe { raw_vec.into_vec() }; +} + +// =============== Forking =============== + +#[no_mangle] +pub extern "C" fn mmtk_before_fork() { + mmtk().prepare_to_fork(); + binding().join_all_gc_threads(); +} + +#[no_mangle] +pub extern "C" fn mmtk_after_fork(tls: VMThread) { + mmtk().after_fork(tls); +} + +// =============== Statistics =============== + +#[no_mangle] +pub extern "C" fn mmtk_total_bytes() -> usize { + memory_manager::total_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_used_bytes() -> usize { + memory_manager::used_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_free_bytes() -> usize { + memory_manager::free_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_starting_heap_address() -> Address { + memory_manager::starting_heap_address() +} + +#[no_mangle] +pub extern "C" fn mmtk_last_heap_address() -> Address { + memory_manager::last_heap_address() +} + +#[no_mangle] +pub extern "C" fn mmtk_worker_count() -> usize { + memory_manager::num_of_workers(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_plan() -> *const u8 { + static NO_GC: &[u8] = b"NoGC\0"; + static MARK_SWEEP: &[u8] = b"MarkSweep\0"; + static IMMIX: &[u8] = b"Immix\0"; + + match *crate::BINDING.get().unwrap().mmtk.get_options().plan { + PlanSelector::NoGC => NO_GC.as_ptr(), + PlanSelector::MarkSweep => MARK_SWEEP.as_ptr(), + PlanSelector::Immix => IMMIX.as_ptr(), + _ => panic!("Unknown plan"), + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_mode() -> *const u8 { + static FIXED_HEAP: &[u8] = b"fixed\0"; + static DYNAMIC_HEAP: &[u8] = b"dynamic\0"; + static RUBY_HEAP: &[u8] = b"ruby\0"; + static CPU_HEAP: &[u8] = b"cpu\0"; + + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(_) => FIXED_HEAP.as_ptr(), + GCTriggerSelector::DynamicHeapSize(_, _) => DYNAMIC_HEAP.as_ptr(), + GCTriggerSelector::Delegated => { + // Two delegated triggers exist; disambiguate via the populated + // config singleton. + if CPU_HEAP_TRIGGER_CONFIG.get().is_some() { + CPU_HEAP.as_ptr() + } else { + RUBY_HEAP.as_ptr() + } + } + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_min() -> usize { + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(_) => 0, + GCTriggerSelector::DynamicHeapSize(min_size, _) => min_size, + GCTriggerSelector::Delegated => { + if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() { + conversions::pages_to_bytes(cfg.min_heap_pages) + } else { + conversions::pages_to_bytes( + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("RUBY_HEAP_TRIGGER_CONFIG not set") + .min_heap_pages, + ) + } + } + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_max() -> usize { + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(max_size) => max_size, + GCTriggerSelector::DynamicHeapSize(_, max_size) => max_size, + GCTriggerSelector::Delegated => { + if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() { + conversions::pages_to_bytes(cfg.max_heap_pages) + } else { + conversions::pages_to_bytes( + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("RUBY_HEAP_TRIGGER_CONFIG not set") + .max_heap_pages, + ) + } + } + } +} + +// =============== Miscellaneous =============== + +#[no_mangle] +pub extern "C" fn mmtk_is_mmtk_object(addr: Address) -> bool { + debug_assert!(!addr.is_zero()); + debug_assert!(addr.is_aligned_to(mmtk::util::is_mmtk_object::VO_BIT_REGION_SIZE)); + memory_manager::is_mmtk_object(addr).is_some() +} diff --git a/gc/mmtk/src/binding.rs b/gc/mmtk/src/binding.rs new file mode 100644 index 0000000000..36d4a992fd --- /dev/null +++ b/gc/mmtk/src/binding.rs @@ -0,0 +1,129 @@ +use std::collections::HashSet; +use std::ffi::CString; +use std::sync::atomic::AtomicBool; +use std::sync::Mutex; +use std::thread::JoinHandle; + +use mmtk::util::ObjectReference; +use mmtk::MMTK; + +use crate::abi; +use crate::abi::RubyBindingOptions; +use crate::pinning_registry::PinningRegistry; +use crate::weak_proc::WeakProcessor; +use crate::Ruby; + +pub struct RubyBindingFast { + pub suffix_size: usize, +} + +impl Default for RubyBindingFast { + fn default() -> Self { + Self::new() + } +} + +impl RubyBindingFast { + pub const fn new() -> Self { + Self { suffix_size: 0 } + } +} + +pub struct RubyConfiguration { + pub gc_enabled: AtomicBool, +} + +impl Default for RubyConfiguration { + fn default() -> Self { + Self::new() + } +} + +impl RubyConfiguration { + pub const fn new() -> Self { + Self { + // Mimic the old behavior when the gc_enabled flag was in mmtk-core. + // We may refactor it so that it is false by default. + gc_enabled: AtomicBool::new(true), + } + } +} + +pub struct RubyBinding { + pub mmtk: &'static MMTK<Ruby>, + pub options: RubyBindingOptions, + pub upcalls: *const abi::RubyUpcalls, + pub plan_name: Mutex<Option<CString>>, + pub weak_proc: WeakProcessor, + pub pinning_registry: PinningRegistry, + pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>, + pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>, +} + +unsafe impl Sync for RubyBinding {} +unsafe impl Send for RubyBinding {} + +impl RubyBinding { + pub fn new( + mmtk: &'static MMTK<Ruby>, + binding_options: &RubyBindingOptions, + upcalls: *const abi::RubyUpcalls, + ) -> Self { + unsafe { + crate::BINDING_FAST.suffix_size = binding_options.suffix_size; + } + + Self { + mmtk, + options: binding_options.clone(), + upcalls, + plan_name: Mutex::new(None), + weak_proc: WeakProcessor::new(), + pinning_registry: PinningRegistry::new(), + gc_thread_join_handles: Default::default(), + wb_unprotected_objects: Default::default(), + } + } + + pub fn upcalls(&self) -> &'static abi::RubyUpcalls { + unsafe { &*self.upcalls as &'static abi::RubyUpcalls } + } + + pub fn get_plan_name_c(&self) -> *const libc::c_char { + let mut plan_name = self.plan_name.lock().unwrap(); + if plan_name.is_none() { + let name_string = format!("{:?}", *self.mmtk.get_options().plan); + let c_string = CString::new(name_string) + .unwrap_or_else(|e| panic!("Failed converting plan name to CString: {e}")); + *plan_name = Some(c_string); + } + plan_name.as_deref().unwrap().as_ptr() + } + + pub fn join_all_gc_threads(&self) { + let handles = { + let mut guard = self.gc_thread_join_handles.lock().unwrap(); + std::mem::take(&mut *guard) + }; + + debug!("Joining GC threads..."); + let total = handles.len(); + let mut joined = 0; + for handle in handles { + handle.join().unwrap(); + joined += 1; + debug!("{joined}/{total} GC threads joined."); + } + } + + pub fn register_wb_unprotected_object(&self, object: ObjectReference) { + debug!("Registering WB-unprotected object: {object}"); + let mut objects = self.wb_unprotected_objects.lock().unwrap(); + objects.insert(object); + } + + pub fn object_wb_unprotected_p(&self, object: ObjectReference) -> bool { + let objects = self.wb_unprotected_objects.lock().unwrap(); + objects.contains(&object) + } +} diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs new file mode 100644 index 0000000000..648efa4e27 --- /dev/null +++ b/gc/mmtk/src/collection.rs @@ -0,0 +1,122 @@ +use crate::abi::GCThreadTLS; + +use crate::api::RubyMutator; +use crate::heap::CpuHeapTrigger; +use crate::heap::RubyHeapTrigger; +use crate::heap::CPU_HEAP_TRIGGER_CONFIG; +use crate::mmtk; +use crate::upcalls; +use crate::Ruby; +use mmtk::memory_manager; +use mmtk::scheduler::*; +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; +use mmtk::util::VMWorkerThread; +use mmtk::vm::Collection; +use mmtk::vm::GCThreadContext; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::thread; + +static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(false); + +pub struct VMCollection {} + +impl Collection<Ruby> for VMCollection { + fn is_collection_enabled() -> bool { + crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed) + } + + fn stop_all_mutators<F>(tls: VMWorkerThread, mut mutator_visitor: F) + where + F: FnMut(&'static mut mmtk::Mutator<Ruby>), + { + (upcalls().stop_the_world)(); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed); + (upcalls().before_updating_jit_code)(); + } else { + CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed); + } + + crate::binding().pinning_registry.pin_children(tls); + + (upcalls().get_mutators)( + Self::notify_mutator_ready::<F>, + &mut mutator_visitor as *mut F as *mut _, + ); + } + + fn resume_mutators(_tls: VMWorkerThread) { + let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed); + + if current_gc_may_move { + (upcalls().after_updating_jit_code)(); + } + + (upcalls().resume_mutators)(current_gc_may_move); + } + + fn block_for_gc(tls: VMMutatorThread) { + (upcalls().block_for_gc)(tls); + } + + fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext<Ruby>) { + let join_handle = match ctx { + GCThreadContext::Worker(mut worker) => thread::Builder::new() + .name("MMTk Worker Thread".to_string()) + .spawn(move || { + let ordinal = worker.ordinal; + debug!("Hello! This is MMTk Worker Thread running! ordinal: {ordinal}"); + crate::register_gc_thread(thread::current().id()); + let ptr_worker = &mut *worker as *mut GCWorker<Ruby>; + let gc_thread_tls = + Box::into_raw(Box::new(GCThreadTLS::for_worker(ptr_worker))); + (upcalls().init_gc_worker_thread)(gc_thread_tls); + memory_manager::start_worker( + mmtk(), + GCThreadTLS::to_vwt(gc_thread_tls), + worker, + ); + debug!("An MMTk Worker Thread is quitting. Good bye! ordinal: {ordinal}"); + crate::unregister_gc_thread(thread::current().id()); + }) + .unwrap(), + }; + + { + let mut handles = crate::binding().gc_thread_join_handles.lock().unwrap(); + handles.push(join_handle); + } + } + + fn vm_live_bytes() -> usize { + (upcalls().vm_live_bytes)() + } + + fn create_gc_trigger() -> Box<dyn GCTriggerPolicy<Ruby>> { + // `GCTriggerSelector::Delegated` is currently used by two different + // heap modes: `ruby` (the Ruby-like free-slot ratio trigger) and `cpu` + // (the CPU-overhead trigger from Tavakolisomeh et al., MPLR '23). + // Which one is active is determined by which `OnceCell` config the + // `MMTK_HEAP_MODE` parser populated. + if CPU_HEAP_TRIGGER_CONFIG.get().is_some() { + Box::new(CpuHeapTrigger::default()) + } else { + Box::new(RubyHeapTrigger::default()) + } + } +} + +impl VMCollection { + extern "C" fn notify_mutator_ready<F>(mutator_ptr: *mut RubyMutator, data: *mut libc::c_void) + where + F: FnMut(&'static mut mmtk::Mutator<Ruby>), + { + let mutator = unsafe { &mut *mutator_ptr }; + let mutator_visitor = unsafe { &mut *(data as *mut F) }; + mutator_visitor(mutator); + } +} diff --git a/gc/mmtk/src/heap/cpu_heap_trigger.rs b/gc/mmtk/src/heap/cpu_heap_trigger.rs new file mode 100644 index 0000000000..ef5a79fe9a --- /dev/null +++ b/gc/mmtk/src/heap/cpu_heap_trigger.rs @@ -0,0 +1,370 @@ +//! A GC trigger that adjusts the heap size based on the CPU overhead of GC. +//! +//! This is an implementation of the heap sizing policy described in +//! Tavakolisomeh, Shimchenko, Österlund, Bruno, Ferreira, Wrigstad, +//! "Heap Size Adjustment with CPU Control", MPLR '23. +//! <https://doi.org/10.1145/3617651.3622988> +//! +//! The idea: rather than letting heap size control GC frequency, let a +//! user-supplied *target GC CPU overhead* control the heap size. After each GC +//! cycle, we measure the GC CPU overhead (fraction of process CPU time spent +//! in GC) and compare it to the target. If GC is over budget we grow the heap +//! (reducing GC frequency); if it is under budget we shrink the heap (trading +//! memory for more frequent collections). +//! +//! ## Algorithm +//! +//! After each GC cycle we compute, using an average of the last `n` cycles: +//! +//! ```text +//! GC_CPU = T_GC / T_APP (Eq. 1) +//! overhead_error = GC_CPU - target (Eq. 2) +//! sigmoid_error = 1 / (1 + e^(-overhead_error)) (Eq. 3) +//! adjustment_factor = sigmoid_error + 0.5 (in (0.5, 1.5)) (Eq. 4) +//! new_size = current_size * adjustment_factor (Eq. 5) +//! ``` +//! +//! where: +//! - `T_GC` is the wall-clock duration of each GC cycle. +//! - `T_APP` is process CPU time elapsed between consecutive GC cycles (sum of +//! CPU time over all threads — mutators, GC workers, compilers, etc.), read +//! via `clock_gettime(CLOCK_PROCESS_CPUTIME_ID)`. +//! +//! The final heap size is then clamped to the range +//! `[max(1.1 * used, min_heap_pages), max_heap_pages]`, providing 10% headroom +//! above current live memory to avoid triggering GC on an effectively-empty +//! heap. +//! +//! ## Differences from the paper +//! +//! The paper targets ZGC, a concurrent generational collector. MMTk's Ruby +//! binding currently ships stop-the-world collectors (Immix, MarkSweep). The +//! paper's formula still applies: with a STW collector the process CPU time +//! during GC closely tracks the wall-clock GC time, and mutator CPU time +//! during the mutator phase is correctly attributed. For generational plans +//! we skip nursery-only GCs, consistent with MemBalancer. + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::heap::SpaceStats; +use mmtk::Plan; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +use crate::Ruby; + +pub static CPU_HEAP_TRIGGER_CONFIG: OnceCell<CpuHeapTriggerConfig> = OnceCell::new(); + +/// Configuration for the [`CpuHeapTrigger`]. +pub struct CpuHeapTriggerConfig { + /// Lower bound on heap size (in pages). The trigger will never shrink below + /// this value. + pub min_heap_pages: usize, + /// Upper bound on heap size (in pages). The trigger will never grow above + /// this value. + pub max_heap_pages: usize, + /// Initial heap size (in pages). + pub initial_heap_pages: usize, + /// Target GC CPU overhead as a fraction of total process CPU time. For + /// example, `0.15` means the policy will try to keep GC CPU usage near 15%. + /// Valid range: `(0.0, 1.0)`. + pub target_gc_cpu: f64, + /// Number of recent GC cycles averaged together when computing the CPU + /// overhead signal. Smoothes out short-term fluctuations. The paper uses 3. + pub window_size: usize, +} + +/// A single GC cycle's timing measurements. +#[derive(Clone, Copy, Debug, Default)] +struct GcSample { + /// Wall-clock seconds spent inside this GC cycle. + gc_seconds: f64, + /// Seconds of process CPU time elapsed since the previous GC cycle ended. + /// This covers both mutator time and (on multi-threaded mutators) any + /// mutator CPU time consumed in parallel with the previous GC. + app_cpu_seconds: f64, +} + +struct CpuHeapTriggerState { + /// Ring buffer of the last `window_size` samples. Oldest-first. + samples: Vec<GcSample>, + /// Wall-clock time when the current GC cycle started. `None` when no GC is + /// in progress. + gc_start_wall: Option<std::time::Instant>, + /// Process CPU time (seconds) recorded at the end of the previous GC + /// cycle. `None` until the first cycle completes. + last_gc_end_cpu: Option<f64>, +} + +impl CpuHeapTriggerState { + fn new() -> Self { + Self { + samples: Vec::new(), + gc_start_wall: None, + last_gc_end_cpu: None, + } + } + + /// Pushes a new sample, dropping the oldest when the window is full. + fn push_sample(&mut self, sample: GcSample, window_size: usize) { + if self.samples.len() >= window_size { + self.samples.remove(0); + } + self.samples.push(sample); + } + + /// Returns the arithmetic mean GC CPU overhead across the window, or + /// `None` if we don't yet have a full sample (which happens on the first + /// GC cycle — we have no baseline for `app_cpu_seconds`). + fn mean_gc_cpu(&self) -> Option<f64> { + if self.samples.is_empty() { + return None; + } + let total_gc: f64 = self.samples.iter().map(|s| s.gc_seconds).sum(); + let total_app: f64 = self.samples.iter().map(|s| s.app_cpu_seconds).sum(); + if total_app <= 0.0 { + return None; + } + Some(total_gc / total_app) + } +} + +pub struct CpuHeapTrigger { + /// Target heap size in pages. Updated at the end of each GC cycle. + target_heap_pages: AtomicUsize, + /// Mutable timing state. Wrapped in a `Mutex` because `on_gc_start` and + /// `on_gc_end` are the only mutation sites and they are not on an + /// allocation hot path; avoiding the complexity of lock-free state is + /// worth the trivial contention. + state: Mutex<CpuHeapTriggerState>, +} + +impl Default for CpuHeapTrigger { + fn default() -> Self { + let cfg = Self::get_config(); + Self { + target_heap_pages: AtomicUsize::new(cfg.initial_heap_pages), + state: Mutex::new(CpuHeapTriggerState::new()), + } + } +} + +impl GCTriggerPolicy<Ruby> for CpuHeapTrigger { + fn is_gc_required( + &self, + space_full: bool, + space: Option<SpaceStats<Ruby>>, + plan: &dyn Plan<VM = Ruby>, + ) -> bool { + // Let the plan decide, matching the other triggers. + plan.collection_required(space_full, space) + } + + fn on_gc_start(&self, _mmtk: &'static MMTK<Ruby>) { + let mut state = self.state.lock().unwrap(); + state.gc_start_wall = Some(std::time::Instant::now()); + } + + fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) { + // Skip nursery-only GCs for generational plans. The heap resizing + // decision is driven by the (much more expensive) full collections + // where the signal-to-noise ratio is high enough to be useful. + if let Some(gen_plan) = mmtk.get_plan().generational() { + if gen_plan.is_current_gc_nursery() { + return; + } + } + + let cfg = Self::get_config(); + let gc_end_cpu = process_cpu_time_seconds(); + + let mut state = self.state.lock().unwrap(); + + // Duration of this GC cycle (wall clock). + let gc_seconds = state + .gc_start_wall + .take() + .map(|start| start.elapsed().as_secs_f64()) + .unwrap_or(0.0); + + // Process CPU time elapsed since the previous GC cycle ended. We + // require at least one previous end timestamp to produce a valid + // sample — without it we cannot compute `T_APP`. + if let (Some(last_end), Some(now)) = (state.last_gc_end_cpu, gc_end_cpu) { + let app_cpu_seconds = (now - last_end).max(0.0); + // Only record non-degenerate samples to avoid poisoning the window + // with zero-time entries from back-to-back GCs. + if app_cpu_seconds > 0.0 { + state.push_sample( + GcSample { + gc_seconds, + app_cpu_seconds, + }, + cfg.window_size, + ); + } + } + state.last_gc_end_cpu = gc_end_cpu; + + // Compute the new heap size only when we have samples to average over. + if let Some(gc_cpu) = state.mean_gc_cpu() { + // Drop the lock before doing the (relatively cheap) math and + // atomic update; nothing below needs the state. + drop(state); + + let overhead_error = gc_cpu - cfg.target_gc_cpu; // Eq. (2) + let sigmoid_error = sigmoid(overhead_error); // Eq. (3) + let adjustment_factor = sigmoid_error + 0.5; // Eq. (4), range (0.5, 1.5) + + let current = self.target_heap_pages.load(Ordering::Relaxed); + let suggested = ((current as f64) * adjustment_factor) as usize; // Eq. (5) + + // Clamp: + // - upper bound: configured max + // - lower bound: max(1.1 * used, min) — 10% headroom above current + // live memory, so we never request a heap so small that GC is + // triggered immediately on return from this one. + let used = mmtk.get_plan().get_used_pages(); + let floor = ((used as f64) * 1.1).ceil() as usize; + let lower = floor.max(cfg.min_heap_pages).min(cfg.max_heap_pages); + let upper = cfg.max_heap_pages; + let new_target = suggested.clamp(lower, upper); + + self.target_heap_pages.store(new_target, Ordering::Relaxed); + + info!( + "CpuHeapTrigger: gc_cpu={:.4} target={:.4} factor={:.4} \ + pages {} -> {} (used={}, clamp=[{}, {}])", + gc_cpu, + cfg.target_gc_cpu, + adjustment_factor, + current, + new_target, + used, + lower, + upper + ); + } + } + + fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool { + plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_current_heap_size_in_pages(&self) -> usize { + self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_max_heap_size_in_pages(&self) -> usize { + Self::get_config().max_heap_pages + } + + fn can_heap_size_grow(&self) -> bool { + self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages + } +} + +impl CpuHeapTrigger { + fn get_config<'b>() -> &'b CpuHeapTriggerConfig { + CPU_HEAP_TRIGGER_CONFIG + .get() + .expect("Attempt to use CPU_HEAP_TRIGGER_CONFIG before it is initialized") + } +} + +/// Standard logistic sigmoid. Returns 0.5 when x == 0, asymptotes to 0 and 1. +fn sigmoid(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) +} + +/// Reads the process-wide CPU time as a floating-point number of seconds, +/// summed across all threads of this process. Returns `None` if the clock +/// query fails (which should be essentially impossible on supported +/// platforms). +fn process_cpu_time_seconds() -> Option<f64> { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: `clock_gettime` writes exactly `sizeof(timespec)` bytes to the + // pointer we pass, which is a valid local stack allocation. + let rc = unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) }; + if rc != 0 { + return None; + } + Some((ts.tv_sec as f64) + (ts.tv_nsec as f64) / 1_000_000_000.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sigmoid_is_well_behaved() { + assert!((sigmoid(0.0) - 0.5).abs() < 1e-12); + assert!(sigmoid(-100.0) < 1e-9); + assert!(sigmoid(100.0) > 1.0 - 1e-9); + // Monotonic. + assert!(sigmoid(-1.0) < sigmoid(0.0)); + assert!(sigmoid(0.0) < sigmoid(1.0)); + } + + #[test] + fn adjustment_factor_is_within_paper_bounds() { + // Eq. (4): adjustment_factor = sigmoid(e) + 0.5 must lie in (0.5, 1.5). + for e in [-10.0_f64, -1.0, 0.0, 1.0, 10.0] { + let f = sigmoid(e) + 0.5; + assert!(f > 0.5 && f < 1.5, "factor {f} out of range for e={e}"); + } + } + + #[test] + fn mean_gc_cpu_is_total_weighted() { + let mut state = CpuHeapTriggerState::new(); + state.push_sample( + GcSample { + gc_seconds: 1.0, + app_cpu_seconds: 10.0, + }, + 3, + ); + state.push_sample( + GcSample { + gc_seconds: 3.0, + app_cpu_seconds: 10.0, + }, + 3, + ); + // (1 + 3) / (10 + 10) = 0.2 + assert!((state.mean_gc_cpu().unwrap() - 0.2).abs() < 1e-12); + } + + #[test] + fn window_drops_oldest() { + let mut state = CpuHeapTriggerState::new(); + for i in 0..5 { + state.push_sample( + GcSample { + gc_seconds: i as f64, + app_cpu_seconds: 1.0, + }, + 3, + ); + } + assert_eq!(state.samples.len(), 3); + // After pushing 0,1,2,3,4 with window 3, we should have [2,3,4]. + assert_eq!(state.samples[0].gc_seconds, 2.0); + assert_eq!(state.samples[2].gc_seconds, 4.0); + } + + #[test] + fn no_sample_without_prior_gc() { + // First GC cycle cannot produce a sample (no previous end time). The + // push happens only when last_gc_end_cpu is Some. + let state = CpuHeapTriggerState::new(); + assert!(state.mean_gc_cpu().is_none()); + } +} diff --git a/gc/mmtk/src/heap/mod.rs b/gc/mmtk/src/heap/mod.rs new file mode 100644 index 0000000000..05a35efb23 --- /dev/null +++ b/gc/mmtk/src/heap/mod.rs @@ -0,0 +1,9 @@ +mod cpu_heap_trigger; +mod ruby_heap_trigger; + +pub use cpu_heap_trigger::CpuHeapTrigger; +pub use cpu_heap_trigger::CpuHeapTriggerConfig; +pub use cpu_heap_trigger::CPU_HEAP_TRIGGER_CONFIG; +pub use ruby_heap_trigger::RubyHeapTrigger; +pub use ruby_heap_trigger::RubyHeapTriggerConfig; +pub use ruby_heap_trigger::RUBY_HEAP_TRIGGER_CONFIG; diff --git a/gc/mmtk/src/heap/ruby_heap_trigger.rs b/gc/mmtk/src/heap/ruby_heap_trigger.rs new file mode 100644 index 0000000000..fe1130043d --- /dev/null +++ b/gc/mmtk/src/heap/ruby_heap_trigger.rs @@ -0,0 +1,105 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::heap::SpaceStats; +use mmtk::Plan; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +use crate::Ruby; + +pub static RUBY_HEAP_TRIGGER_CONFIG: OnceCell<RubyHeapTriggerConfig> = OnceCell::new(); + +pub struct RubyHeapTriggerConfig { + /// Min heap size + pub min_heap_pages: usize, + /// Max heap size + pub max_heap_pages: usize, + /// Minimum ratio of empty space after a GC before the heap will grow + pub heap_pages_min_ratio: f64, + /// Ratio the heap will grow by + pub heap_pages_goal_ratio: f64, + /// Maximum ratio of empty space after a GC before the heap will shrink + pub heap_pages_max_ratio: f64, +} + +pub struct RubyHeapTrigger { + /// Target number of heap pages + target_heap_pages: AtomicUsize, +} + +impl GCTriggerPolicy<Ruby> for RubyHeapTrigger { + fn is_gc_required( + &self, + space_full: bool, + space: Option<SpaceStats<Ruby>>, + plan: &dyn Plan<VM = Ruby>, + ) -> bool { + // Let the plan decide + plan.collection_required(space_full, space) + } + + fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) { + if let Some(plan) = mmtk.get_plan().generational() { + if plan.is_current_gc_nursery() { + // Nursery GC + } else { + // Full GC + } + + panic!("TODO: support for generational GC not implemented") + } else { + let used_pages = mmtk.get_plan().get_used_pages(); + + let target_min = + (used_pages as f64 * (1.0 + Self::get_config().heap_pages_min_ratio)) as usize; + let target_max = + (used_pages as f64 * (1.0 + Self::get_config().heap_pages_max_ratio)) as usize; + let new_target = + (((used_pages as f64) * (1.0 + Self::get_config().heap_pages_goal_ratio)) as usize) + .clamp( + Self::get_config().min_heap_pages, + Self::get_config().max_heap_pages, + ); + + if used_pages < target_min || used_pages > target_max { + self.target_heap_pages.store(new_target, Ordering::Relaxed); + } + } + } + + fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool { + plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_current_heap_size_in_pages(&self) -> usize { + self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_max_heap_size_in_pages(&self) -> usize { + Self::get_config().max_heap_pages + } + + fn can_heap_size_grow(&self) -> bool { + self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages + } +} + +impl Default for RubyHeapTrigger { + fn default() -> Self { + let min_heap_pages = Self::get_config().min_heap_pages; + + Self { + target_heap_pages: AtomicUsize::new(min_heap_pages), + } + } +} + +impl RubyHeapTrigger { + fn get_config<'b>() -> &'b RubyHeapTriggerConfig { + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("Attempt to use RUBY_HEAP_TRIGGER_CONFIG before it is initialized") + } +} diff --git a/gc/mmtk/src/lib.rs b/gc/mmtk/src/lib.rs new file mode 100644 index 0000000000..52dc782051 --- /dev/null +++ b/gc/mmtk/src/lib.rs @@ -0,0 +1,161 @@ +// Warn about unsafe operations in functions that are already marked as unsafe. +// This will become default in Rust 2024 edition. +#![warn(unsafe_op_in_unsafe_fn)] + +extern crate libc; +extern crate mmtk; +#[macro_use] +extern crate log; +extern crate probe; + +use std::collections::HashSet; +use std::panic::PanicHookInfo; +use std::sync::Mutex; +use std::thread::ThreadId; + +use abi::RubyUpcalls; +use binding::RubyBinding; +use binding::RubyBindingFast; +use binding::RubyConfiguration; +use mmtk::vm::slot::SimpleSlot; +use mmtk::vm::slot::UnimplementedMemorySlice; +use mmtk::vm::VMBinding; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +pub mod abi; +pub mod active_plan; +pub mod api; +pub mod binding; +pub mod collection; +pub mod heap; +pub mod object_model; +pub mod pinning_registry; +pub mod reference_glue; +pub mod scanning; +pub mod utils; +pub mod weak_proc; + +#[derive(Default)] +pub struct Ruby; + +/// Ruby slot type, i.e. a slot that holds a VALUE. +/// Currently we use SimpleSlot. +/// It doesn't matter, becaues we have not started using slot-enqueuing, yet. +pub type RubySlot = SimpleSlot; + +/// Ruby memory slice, i.e. an array of VALUEs. +/// It is used by array-copy barriers which is supposed to perform bettern than copying array +/// elements one by one. At this moment, we just leave it unimplemented. +pub type RubyMemorySlice = UnimplementedMemorySlice<RubySlot>; + +impl VMBinding for Ruby { + type VMObjectModel = object_model::VMObjectModel; + type VMScanning = scanning::VMScanning; + type VMCollection = collection::VMCollection; + type VMActivePlan = active_plan::VMActivePlan; + type VMReferenceGlue = reference_glue::VMReferenceGlue; + + type VMSlot = RubySlot; + type VMMemorySlice = RubyMemorySlice; +} + +/// The callback for mutator thread panic handler (which calls rb_bug to output +/// debugging information such as the Ruby backtrace and memory maps). +/// This is set before BINDING is set because mmtk_init could panic. +pub static MUTATOR_THREAD_PANIC_HANDLER: OnceCell<extern "C" fn()> = OnceCell::new(); + +/// The singleton object for the Ruby binding itself. +pub static BINDING: OnceCell<RubyBinding> = OnceCell::new(); + +/// Some data needs to be accessed fast. +/// We sacrifice safety for speed using unsynchronized global variables. +pub static mut BINDING_FAST: RubyBindingFast = RubyBindingFast::new(); + +/// Some data needs to be accessed fast. +pub static CONFIGURATION: RubyConfiguration = RubyConfiguration::new(); + +pub fn binding<'b>() -> &'b RubyBinding { + BINDING + .get() + .expect("Attempt to use the binding before it is initialization") +} + +pub fn mmtk() -> &'static MMTK<Ruby> { + binding().mmtk +} + +pub fn upcalls() -> &'static RubyUpcalls { + binding().upcalls() +} + +pub static GC_THREADS: OnceCell<Mutex<HashSet<ThreadId>>> = OnceCell::new(); + +pub(crate) fn register_gc_thread(thread_id: ThreadId) { + let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.insert(thread_id); +} + +pub(crate) fn unregister_gc_thread(thread_id: ThreadId) { + let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.remove(&thread_id); +} + +pub(crate) fn is_gc_thread(thread_id: ThreadId) -> bool { + let gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.contains(&thread_id) +} + +fn handle_gc_thread_panic(panic_info: &PanicHookInfo) { + eprintln!("ERROR: An MMTk GC thread panicked. This is a bug."); + eprintln!("{panic_info}"); + + let bt = std::backtrace::Backtrace::capture(); + match bt.status() { + std::backtrace::BacktraceStatus::Unsupported => { + eprintln!("Backtrace is unsupported.") + } + std::backtrace::BacktraceStatus::Disabled => { + eprintln!("Backtrace is disabled."); + eprintln!("run with `RUST_BACKTRACE=1` environment variable to display a backtrace"); + } + std::backtrace::BacktraceStatus::Captured => { + eprintln!("{bt}"); + } + s => { + eprintln!("Unknown backtrace status: {s:?}"); + } + } +} + +pub(crate) fn set_panic_hook() { + if GC_THREADS.set(Default::default()).is_err() { + return; + } + + let old_hook = std::panic::take_hook(); + + std::panic::set_hook(Box::new(move |panic_info| { + if is_gc_thread(std::thread::current().id()) { + handle_gc_thread_panic(panic_info); + + (crate::binding().upcalls().gc_thread_panic_handler)(); + } else { + old_hook(panic_info); + (crate::MUTATOR_THREAD_PANIC_HANDLER + .get() + .expect("MUTATOR_THREAD_PANIC_HANDLER is not set"))(); + } + })); +} + +/// This kind of assertion is enabled if either building in debug mode or the +/// "extra_assert" feature is enabled. +#[macro_export] +macro_rules! extra_assert { + ($($arg:tt)*) => { + if std::cfg!(any(debug_assertions, feature = "extra_assert")) { + std::assert!($($arg)*); + } + }; +} diff --git a/gc/mmtk/src/object_model.rs b/gc/mmtk/src/object_model.rs new file mode 100644 index 0000000000..d673ca11a0 --- /dev/null +++ b/gc/mmtk/src/object_model.rs @@ -0,0 +1,124 @@ +use std::ptr::copy_nonoverlapping; + +use crate::abi; +use crate::abi::RubyObjectAccess; +use crate::abi::MIN_OBJ_ALIGN; +use crate::abi::OBJREF_OFFSET; +use crate::Ruby; +use mmtk::util::constants::BITS_IN_BYTE; +use mmtk::util::copy::CopySemantics; +use mmtk::util::copy::GCWorkerCopyContext; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::vm::*; + +pub struct VMObjectModel {} + +impl VMObjectModel { + const OBJREF_OFFSET: usize = abi::OBJREF_OFFSET; +} + +impl ObjectModel<Ruby> for VMObjectModel { + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); + + // We overwrite the prepended word which were used to hold object sizes. + const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = + VMLocalForwardingPointerSpec::in_header(-((OBJREF_OFFSET * BITS_IN_BYTE) as isize)); + + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = + VMLocalForwardingBitsSpec::side_first(); + + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = + VMLocalMarkBitSpec::side_after(Self::LOCAL_FORWARDING_BITS_SPEC.as_spec()); + + const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = + VMLocalPinningBitSpec::side_after(Self::LOCAL_MARK_BIT_SPEC.as_spec()); + + const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = + VMLocalLOSMarkNurserySpec::side_after(Self::LOCAL_PINNING_BIT_SPEC.as_spec()); + + const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false; + const OBJECT_REF_OFFSET_LOWER_BOUND: isize = Self::OBJREF_OFFSET as isize; + + const NEED_VO_BITS_DURING_TRACING: bool = true; + + fn copy( + from: ObjectReference, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext<Ruby>, + ) -> ObjectReference { + let from_acc = RubyObjectAccess::from_objref(from); + let from_start = from_acc.obj_start(); + let object_size = from_acc.object_size(); + let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics); + debug_assert!(!to_start.is_zero()); + let to_payload = to_start.add(OBJREF_OFFSET); + unsafe { + copy_nonoverlapping::<u8>(from_start.to_ptr(), to_start.to_mut_ptr(), object_size); + } + let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(to_payload) }; + copy_context.post_copy(to_obj, object_size, semantics); + trace!("Copied object from {} to {}", from, to_obj); + + (crate::binding().upcalls().move_obj_during_marking)(from, to_obj); + + #[cfg(feature = "clear_old_copy")] + { + trace!( + "Clearing old copy {} ({}-{})", + from, + from_start, + from_start + object_size + ); + // For debug purpose, we clear the old copy so that if the Ruby VM reads from the old + // copy again, it will likely result in an error. + unsafe { std::ptr::write_bytes::<u8>(from_start.to_mut_ptr(), 0, object_size) } + } + + to_obj + } + + fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address { + unimplemented!( + "This function cannot be called because we do not support MarkCompact for Ruby." + ) + } + + fn get_reference_when_copied_to(_from: ObjectReference, _to: Address) -> ObjectReference { + unimplemented!( + "This function cannot be called because we do not support MarkCompact for Ruby." + ) + } + + fn get_current_size(object: ObjectReference) -> usize { + RubyObjectAccess::from_objref(object).object_size() + } + + fn get_type_descriptor(_reference: ObjectReference) -> &'static [i8] { + todo!() + } + + fn ref_to_object_start(object: ObjectReference) -> Address { + RubyObjectAccess::from_objref(object).obj_start() + } + + fn ref_to_header(object: ObjectReference) -> Address { + RubyObjectAccess::from_objref(object).payload_addr() + } + + fn get_size_when_copied(object: ObjectReference) -> usize { + Self::get_current_size(object) + } + + fn get_align_when_copied(_object: ObjectReference) -> usize { + todo!() + } + + fn get_align_offset_when_copied(_object: ObjectReference) -> usize { + todo!() + } + + fn dump_object(_object: ObjectReference) { + todo!() + } +} diff --git a/gc/mmtk/src/pinning_registry.rs b/gc/mmtk/src/pinning_registry.rs new file mode 100644 index 0000000000..b498b508f1 --- /dev/null +++ b/gc/mmtk/src/pinning_registry.rs @@ -0,0 +1,187 @@ +use std::sync::Mutex; + +use mmtk::memory_manager; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::MMTK; + +use crate::abi::GCThreadTLS; +use crate::upcalls; +use crate::Ruby; + +pub struct PinningRegistry { + pinning_objs: Mutex<Vec<ObjectReference>>, + pinned_objs: Mutex<Vec<ObjectReference>>, +} + +impl PinningRegistry { + pub fn new() -> Self { + Self { + pinning_objs: Default::default(), + pinned_objs: Default::default(), + } + } + + pub fn register(&self, object: ObjectReference) { + let mut pinning_objs = self.pinning_objs.lock().unwrap(); + pinning_objs.push(object); + } + + pub fn pin_children(&self, tls: VMWorkerThread) { + if !crate::mmtk().get_plan().current_gc_may_move_object() { + log::debug!("The current GC is non-moving, skipping pinning children."); + return; + } + + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let worker = gc_tls.worker(); + + let pinning_objs = self + .pinning_objs + .try_lock() + .expect("PinningRegistry should not have races during GC."); + + let packet_size = 512; + let work_packets = pinning_objs + .chunks(packet_size) + .map(|chunk| { + Box::new(PinPinningChildren { + pinning_objs: chunk.to_vec(), + }) as _ + }) + .collect(); + + worker.scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(work_packets); + } + + pub fn cleanup(&self, worker: &mut GCWorker<Ruby>) { + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(RemoveDeadPinnings); + if crate::mmtk().get_plan().current_gc_may_move_object() { + let packet = { + let mut pinned_objs = self + .pinned_objs + .try_lock() + .expect("Unexpected contention on pinned_objs"); + UnpinPinnedObjects { + objs: std::mem::take(&mut pinned_objs), + } + }; + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(packet); + } else { + debug!("The current GC is non-moving, skipping unpinning objects."); + debug_assert_eq!( + { + let pinned_objs = self + .pinned_objs + .try_lock() + .expect("Unexpected contention on pinned_objs"); + pinned_objs.len() + }, + 0 + ); + } + } +} + +impl Default for PinningRegistry { + fn default() -> Self { + Self::new() + } +} + +struct PinPinningChildren { + pinning_objs: Vec<ObjectReference>, +} + +impl GCWork<Ruby> for PinPinningChildren { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + let mut pinned_objs = vec![]; + let mut newly_pinned_objs = vec![]; + + let visit_object = |_worker, target_object: ObjectReference, pin| { + log::trace!( + " -> {} {}", + if pin { "(pin)" } else { " " }, + target_object + ); + if pin { + debug_assert!( + target_object.get_forwarded_object().is_none(), + "Trying to pin {target_object} but has been moved" + ); + + pinned_objs.push(target_object); + } + target_object + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + for obj in self.pinning_objs.iter().cloned() { + log::trace!(" Pinning: {}", obj); + (upcalls().call_gc_mark_children)(obj); + } + }); + + for target_object in pinned_objs { + if memory_manager::pin_object(target_object) { + newly_pinned_objs.push(target_object); + } + } + + let mut pinned_objs = crate::binding() + .pinning_registry + .pinned_objs + .lock() + .unwrap(); + pinned_objs.append(&mut newly_pinned_objs); + } +} + +struct RemoveDeadPinnings; + +impl GCWork<Ruby> for RemoveDeadPinnings { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + log::debug!("Removing dead Pinnings..."); + + let registry = &crate::binding().pinning_registry; + { + let mut pinning_objs = registry + .pinning_objs + .try_lock() + .expect("PinningRegistry should not have races during GC."); + + pinning_objs.retain_mut(|obj| { + if obj.is_live() { + let new_obj = obj.get_forwarded_object().unwrap_or(*obj); + *obj = new_obj; + true + } else { + log::trace!(" Dead Pinning removed: {}", *obj); + false + } + }); + } + } +} + +struct UnpinPinnedObjects { + objs: Vec<ObjectReference>, +} + +impl GCWork<Ruby> for UnpinPinnedObjects { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + log::debug!("Unpinning pinned objects..."); + + for obj in self.objs.iter() { + let unpinned = memory_manager::unpin_object(*obj); + debug_assert!(unpinned); + } + } +} diff --git a/gc/mmtk/src/reference_glue.rs b/gc/mmtk/src/reference_glue.rs new file mode 100644 index 0000000000..1272bd54c1 --- /dev/null +++ b/gc/mmtk/src/reference_glue.rs @@ -0,0 +1,26 @@ +use crate::Ruby; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::vm::ReferenceGlue; + +pub struct VMReferenceGlue {} + +impl ReferenceGlue<Ruby> for VMReferenceGlue { + type FinalizableType = ObjectReference; + + fn get_referent(_object: ObjectReference) -> Option<ObjectReference> { + unimplemented!() + } + + fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { + unimplemented!() + } + + fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) { + unimplemented!() + } + + fn clear_referent(_new_reference: ObjectReference) { + unimplemented!() + } +} diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs new file mode 100644 index 0000000000..355a2e7759 --- /dev/null +++ b/gc/mmtk/src/scanning.rs @@ -0,0 +1,291 @@ +use crate::abi::GCThreadTLS; + +use crate::upcalls; +use crate::utils::ChunkedVecCollector; +use crate::Ruby; +use crate::RubySlot; +use mmtk::memory_manager; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::vm::ObjectTracer; +use mmtk::vm::RootsWorkFactory; +use mmtk::vm::Scanning; +use mmtk::vm::SlotVisitor; +use mmtk::Mutator; + +pub struct VMScanning {} + +impl Scanning<Ruby> for VMScanning { + const UNIQUE_OBJECT_ENQUEUING: bool = true; + + fn support_slot_enqueuing(_tls: VMWorkerThread, _object: ObjectReference) -> bool { + false + } + + fn scan_object<EV: SlotVisitor<RubySlot>>( + _tls: VMWorkerThread, + _object: ObjectReference, + _slot_visitor: &mut EV, + ) { + unreachable!("We have not enabled slot enqueuing for any types, yet."); + } + + fn scan_object_and_trace_edges<OT: ObjectTracer>( + tls: VMWorkerThread, + object: ObjectReference, + object_tracer: &mut OT, + ) { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "Not an MMTk object: {object}", + ); + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let visit_object = |_worker, target_object: ObjectReference, pin| { + trace!( + "Tracing edge: {} -> {}{}", + object, + target_object, + if pin { " pin" } else { "" } + ); + debug_assert!( + mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(), + "Destination is not an MMTk object. Src: {object} dst: {target_object}" + ); + + debug_assert!( + // If we are in a moving GC, all objects should be pinned by PinningRegistry. + // If it is requested that target_object be pinned but it is not pinned, then + // it is a bug because it could be moved. + if crate::mmtk().get_plan().current_gc_may_move_object() && pin { + memory_manager::is_pinned(target_object) + } else { + true + }, + "Object {object} is trying to pin {target_object}" + ); + + let forwarded_target = object_tracer.trace_object(target_object); + if forwarded_target != target_object { + trace!(" Forwarded target {target_object} -> {forwarded_target}"); + } + forwarded_target + }; + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + (upcalls().call_gc_mark_children)(object); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + (upcalls().update_object_references)(object); + } + }); + } + + fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { + // Do nothing + } + + fn scan_roots_in_mutator_thread( + _tls: VMWorkerThread, + _mutator: &'static mut Mutator<Ruby>, + mut _factory: impl RootsWorkFactory<RubySlot>, + ) { + // Do nothing. All stacks (including Ruby stacks and machine stacks) are reachable from + // `rb_vm_t` -> ractor -> thread -> fiber -> stacks. It is part of `ScanGCRoots` which + // calls `rb_gc_mark_roots` -> `rb_vm_mark`. + } + + fn scan_vm_specific_roots(tls: VMWorkerThread, factory: impl RootsWorkFactory<RubySlot>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let root_scanning_work_packets: Vec<Box<dyn GCWork<Ruby>>> = vec![ + Box::new(ScanGCRoots::new(factory.clone())), + Box::new(ScanObjspace::new(factory.clone())), + ]; + gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare] + .bulk_add(root_scanning_work_packets); + + // Generate WB-unprotected roots scanning work packets + + 'gen_wb_unprotected_work: { + let is_nursery_gc = (crate::mmtk().get_plan().generational()) + .is_some_and(|gen| gen.is_current_gc_nursery()); + if !is_nursery_gc { + break 'gen_wb_unprotected_work; + } + + let vecs = { + let guard = crate::binding() + .wb_unprotected_objects + .try_lock() + .expect("Someone is holding the lock of wb_unprotected_objects?"); + if guard.is_empty() { + break 'gen_wb_unprotected_work; + } + + let mut collector = ChunkedVecCollector::new(128); + collector.extend(guard.iter().copied()); + collector.into_vecs() + }; + + let packets = vecs + .into_iter() + .map(|objects| { + let factory = factory.clone(); + Box::new(ScanWbUnprotectedRoots { factory, objects }) as _ + }) + .collect::<Vec<_>>(); + + gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(packets); + } + } + + fn supports_return_barrier() -> bool { + false + } + + fn prepare_for_roots_re_scanning() { + todo!() + } + + fn process_weak_refs( + worker: &mut GCWorker<Ruby>, + tracer_context: impl mmtk::vm::ObjectTracerContext<Ruby>, + ) -> bool { + crate::binding() + .weak_proc + .process_weak_stuff(worker, tracer_context); + crate::binding().pinning_registry.cleanup(worker); + false + } + + fn forward_weak_refs( + _worker: &mut GCWorker<Ruby>, + _tracer_context: impl mmtk::vm::ObjectTracerContext<Ruby>, + ) { + panic!("We can't use MarkCompact in Ruby."); + } +} + +impl VMScanning { + const OBJECT_BUFFER_SIZE: usize = 4096; + + fn collect_object_roots_in<F: FnOnce()>( + root_scan_kind: &str, + gc_tls: &mut GCThreadTLS, + factory: &mut impl RootsWorkFactory<RubySlot>, + callback: F, + ) { + let mut buffer: Vec<ObjectReference> = Vec::new(); + let visit_object = |_, object: ObjectReference, pin| { + debug!( + "[{}] Visiting object: {}{}", + root_scan_kind, + object, + if pin { + "(unmovable root)" + } else { + "(movable, but we pin it anyway)" + } + ); + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "Root does not point to MMTk object. object: {object}" + ); + buffer.push(object); + if buffer.len() >= Self::OBJECT_BUFFER_SIZE { + factory.create_process_pinning_roots_work(std::mem::take(&mut buffer)); + } + object + }; + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, callback); + + if !buffer.is_empty() { + factory.create_process_pinning_roots_work(buffer); + } + } +} + +trait GlobaRootScanningWork { + type F: RootsWorkFactory<RubySlot>; + const NAME: &'static str; + + fn new(factory: Self::F) -> Self; + fn scan_roots(); + fn roots_work_factory(&mut self) -> &mut Self::F; + + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + let factory = self.roots_work_factory(); + + VMScanning::collect_object_roots_in(Self::NAME, gc_tls, factory, || { + Self::scan_roots(); + }); + } +} + +macro_rules! define_global_root_scanner { + ($name: ident, $code: expr) => { + struct $name<F: RootsWorkFactory<RubySlot>> { + factory: F, + } + impl<F: RootsWorkFactory<RubySlot>> GlobaRootScanningWork for $name<F> { + type F = F; + const NAME: &'static str = stringify!($name); + fn new(factory: Self::F) -> Self { + Self { factory } + } + fn scan_roots() { + $code + } + fn roots_work_factory(&mut self) -> &mut Self::F { + &mut self.factory + } + } + impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for $name<F> { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobaRootScanningWork::do_work(self, worker, mmtk); + } + } + }; +} + +define_global_root_scanner!(ScanGCRoots, { + (crate::upcalls().scan_gc_roots)(); +}); + +define_global_root_scanner!(ScanObjspace, { + (crate::upcalls().scan_objspace)(); +}); + +struct ScanWbUnprotectedRoots<F: RootsWorkFactory<RubySlot>> { + factory: F, + objects: Vec<ObjectReference>, +} + +impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for ScanWbUnprotectedRoots<F> { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + VMScanning::collect_object_roots_in("wb_unprot_roots", gc_tls, &mut self.factory, || { + for object in self.objects.iter().copied() { + if object.is_reachable() { + debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}"); + (upcalls().call_gc_mark_children)(object); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + (upcalls().update_object_references)(object); + } + } else { + debug!( + "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {object}" + ); + } + } + }); + } +} diff --git a/gc/mmtk/src/utils.rs b/gc/mmtk/src/utils.rs new file mode 100644 index 0000000000..d1979eaf58 --- /dev/null +++ b/gc/mmtk/src/utils.rs @@ -0,0 +1,161 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use atomic_refcell::AtomicRefCell; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; + +use crate::Ruby; +use sysinfo::System; + +pub struct ChunkedVecCollector<T> { + vecs: Vec<Vec<T>>, + current_vec: Vec<T>, + chunk_size: usize, +} + +impl<T> ChunkedVecCollector<T> { + pub fn new(chunk_size: usize) -> Self { + Self { + vecs: vec![], + current_vec: Vec::with_capacity(chunk_size), + chunk_size, + } + } + + pub fn add(&mut self, item: T) { + self.current_vec.push(item); + if self.current_vec.len() == self.chunk_size { + self.flush(); + } + } + + fn flush(&mut self) { + let new_vec = Vec::with_capacity(self.chunk_size); + let old_vec = std::mem::replace(&mut self.current_vec, new_vec); + self.vecs.push(old_vec); + } + + pub fn into_vecs(mut self) -> Vec<Vec<T>> { + if !self.current_vec.is_empty() { + self.flush(); + } + self.vecs + } +} + +impl<A> Extend<A> for ChunkedVecCollector<A> { + fn extend<T: IntoIterator<Item = A>>(&mut self, iter: T) { + for item in iter { + self.add(item); + } + } +} + +pub struct AfterAll { + counter: AtomicUsize, + stage: WorkBucketStage, + packets: AtomicRefCell<Vec<Box<dyn GCWork<Ruby>>>>, +} + +unsafe impl Sync for AfterAll {} + +impl AfterAll { + pub fn new(stage: WorkBucketStage) -> Self { + Self { + counter: AtomicUsize::new(0), + stage, + packets: AtomicRefCell::new(vec![]), + } + } + + pub fn add_packets(&self, mut packets: Vec<Box<dyn GCWork<Ruby>>>) { + let mut borrow = self.packets.borrow_mut(); + borrow.append(&mut packets); + } + + pub fn count_up(&self, n: usize) { + self.counter.fetch_add(n, Ordering::SeqCst); + } + + pub fn count_down(&self, worker: &mut GCWorker<Ruby>) { + let old = self.counter.fetch_sub(1, Ordering::SeqCst); + if old == 1 { + let packets = { + let mut borrow = self.packets.borrow_mut(); + std::mem::take(borrow.as_mut()) + }; + worker.scheduler().work_buckets[self.stage].bulk_add(packets); + } + } +} + +pub fn default_heap_max() -> usize { + let mut s = System::new(); + s.refresh_memory(); + s.total_memory() + .checked_mul(80) + .and_then(|v| v.checked_div(100)) + .expect("Invalid Memory size") as usize +} + +pub fn parse_capacity(input: &str) -> Option<usize> { + let trimmed = input.trim(); + + const KIBIBYTE: usize = 1024; + const MEBIBYTE: usize = 1024 * KIBIBYTE; + const GIBIBYTE: usize = 1024 * MEBIBYTE; + + let (number, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) { + trimmed.split_at(pos) + } else { + (trimmed, "") + }; + + let Ok(v) = number.parse::<usize>() else { + return None; + }; + + match suffix { + "GiB" => Some(v * GIBIBYTE), + "MiB" => Some(v * MEBIBYTE), + "KiB" => Some(v * KIBIBYTE), + "" => Some(v), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_capacity_parses_bare_bytes() { + assert_eq!(Some(1234), parse_capacity("1234")); + } + + #[test] + fn test_parse_capacity_parses_kibibytes() { + assert_eq!(Some(10240), parse_capacity("10KiB")); + } + + #[test] + fn test_parse_capacity_parses_mebibytes() { + assert_eq!(Some(10485760), parse_capacity("10MiB")) + } + + #[test] + fn test_parse_capacity_parses_gibibytes() { + assert_eq!(Some(10737418240), parse_capacity("10GiB")) + } + + #[test] + fn test_parse_capacity_parses_nonsense_values() { + assert_eq!(None, parse_capacity("notanumber")); + assert_eq!(None, parse_capacity("5tartswithanumber")); + assert_eq!(None, parse_capacity("number1nthemiddle")); + assert_eq!(None, parse_capacity("numberattheend111")); + assert_eq!(None, parse_capacity("mult1pl3numb3r5")); + } +} diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs new file mode 100644 index 0000000000..d38dbe04a4 --- /dev/null +++ b/gc/mmtk/src/weak_proc.rs @@ -0,0 +1,328 @@ +use std::sync::Mutex; + +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::vm::ObjectTracerContext; + +use crate::abi::GCThreadTLS; +use crate::upcalls; +use crate::Ruby; + +pub struct WeakProcessor { + non_parallel_obj_free_candidates: Mutex<Vec<ObjectReference>>, + parallel_obj_free_candidates: Vec<Mutex<Vec<ObjectReference>>>, + + /// Objects that needs `obj_free` called when dying. + /// If it is a bottleneck, replace it with a lock-free data structure, + /// or add candidates in batch. + weak_references: Mutex<Vec<ObjectReference>>, +} + +impl Default for WeakProcessor { + fn default() -> Self { + Self::new() + } +} + +impl WeakProcessor { + pub fn new() -> Self { + Self { + non_parallel_obj_free_candidates: Mutex::new(Vec::new()), + parallel_obj_free_candidates: vec![Mutex::new(Vec::new())], + weak_references: Mutex::new(Vec::new()), + } + } + + pub fn init_parallel_obj_free_candidates(&mut self, num_workers: usize) { + debug_assert_eq!(self.parallel_obj_free_candidates.len(), 1); + + for _ in 1..num_workers { + self.parallel_obj_free_candidates + .push(Mutex::new(Vec::new())); + } + } + + /// Add a batch of objects as candidates for `obj_free`. + /// + /// Amortizes mutex acquisition over the entire batch. Called when a + /// mutator's local buffer is flushed (buffer full or stop-the-world). + pub fn add_obj_free_candidates_batch( + &self, + objects: &[ObjectReference], + can_parallel_free: bool, + ) { + if objects.is_empty() { + return; + } + + if can_parallel_free { + let num_buckets = self.parallel_obj_free_candidates.len(); + for idx in 0..num_buckets { + let mut bucket = self.parallel_obj_free_candidates[idx].lock().unwrap(); + for (i, &obj) in objects.iter().enumerate() { + if i % num_buckets == idx { + bucket.push(obj); + } + } + } + } else { + self.non_parallel_obj_free_candidates + .lock() + .unwrap() + .extend_from_slice(objects); + } + } + + pub fn get_all_obj_free_candidates(&self) -> Vec<ObjectReference> { + // let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap(); + let mut all_obj_free_candidates = self + .non_parallel_obj_free_candidates + .lock() + .unwrap() + .to_vec(); + + for candidates_mutex in &self.parallel_obj_free_candidates { + all_obj_free_candidates.extend(candidates_mutex.lock().unwrap().to_vec()); + } + + std::mem::take(all_obj_free_candidates.as_mut()) + } + + pub fn add_weak_reference(&self, object: ObjectReference) { + let mut weak_references = self.weak_references.lock().unwrap(); + weak_references.push(object); + } + + pub fn weak_references_count(&self) -> usize { + self.weak_references.lock().unwrap().len() + } + + pub fn process_weak_stuff( + &self, + worker: &mut GCWorker<Ruby>, + _tracer_context: impl ObjectTracerContext<Ruby>, + ) { + worker.add_work( + WorkBucketStage::VMRefClosure, + ProcessNonParallelObjFreeCanadidates {}, + ); + + for index in 0..self.parallel_obj_free_candidates.len() { + worker.add_work( + WorkBucketStage::VMRefClosure, + ProcessParallelObjFreeCandidates { index }, + ); + } + + worker.add_work(WorkBucketStage::VMRefClosure, ProcessWeakReferences); + + worker.add_work(WorkBucketStage::Prepare, UpdateFinalizerObjIdTables); + + let global_tables_count = (crate::upcalls().global_tables_count)(); + let work_packets = (0..global_tables_count) + .map(|i| Box::new(UpdateGlobalTables { idx: i }) as _) + .collect(); + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(work_packets); + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure] + .bulk_add(vec![Box::new(UpdateWbUnprotectedObjectsList) as _]); + } +} + +fn process_obj_free_candidates(obj_free_candidates: &mut Vec<ObjectReference>) { + // Process obj_free + let mut new_candidates = Vec::new(); + + for object in obj_free_candidates.iter().copied() { + if object.is_reachable() { + // Forward and add back to the candidate list. + let new_object = object.forward(); + trace!("Forwarding obj_free candidate: {object} -> {new_object}"); + new_candidates.push(new_object); + } else { + (upcalls().call_obj_free)(object); + } + } + + *obj_free_candidates = new_candidates; +} + +struct ProcessParallelObjFreeCandidates { + index: usize, +} + +impl GCWork<Ruby> for ProcessParallelObjFreeCandidates { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut obj_free_candidates = crate::binding().weak_proc.parallel_obj_free_candidates + [self.index] + .try_lock() + .expect("Lock for parallel_obj_free_candidates should not be held"); + + process_obj_free_candidates(&mut obj_free_candidates); + } +} + +struct ProcessNonParallelObjFreeCanadidates; + +impl GCWork<Ruby> for ProcessNonParallelObjFreeCanadidates { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut obj_free_candidates = crate::binding() + .weak_proc + .non_parallel_obj_free_candidates + .try_lock() + .expect("Lock for non_parallel_obj_free_candidates should not be held"); + + process_obj_free_candidates(&mut obj_free_candidates); + } +} + +struct ProcessWeakReferences; + +impl GCWork<Ruby> for ProcessWeakReferences { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + if crate::mmtk().get_plan().current_gc_may_move_object() { + let gc_tls: &mut GCThreadTLS = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + let visit_object = |_worker, target_object: ObjectReference, _pin| { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(), + "Destination is not an MMTk object" + ); + + target_object + .get_forwarded_object() + .unwrap_or(target_object) + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + self.process_weak_references(true); + }) + } else { + self.process_weak_references(false); + } + } +} + +impl ProcessWeakReferences { + fn process_weak_references(&mut self, moving_gc: bool) { + let mut weak_references = crate::binding() + .weak_proc + .weak_references + .try_lock() + .expect("Mutators should not be holding the lock."); + + weak_references.retain_mut(|object_ptr| { + let object = object_ptr.get_forwarded_object().unwrap_or(*object_ptr); + + if object != *object_ptr { + *object_ptr = object; + } + + if object.is_reachable() { + (upcalls().handle_weak_references)(object, moving_gc); + + true + } else { + false + } + }); + } +} + +trait GlobalTableProcessingWork { + fn process_table(&mut self); + + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + // `hash_foreach_replace` depends on `gb_object_moved_p` which has to have the semantics + // of `trace_object` due to the way it is used in `UPDATE_IF_MOVED`. + let forward_object = |_worker, object: ObjectReference, _pin| { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "{object} is not an MMTk object" + ); + let result = object.forward(); + trace!("Forwarding reference: {object} -> {result}"); + result + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(forward_object, || { + self.process_table(); + }); + } +} + +struct UpdateFinalizerObjIdTables; +impl GlobalTableProcessingWork for UpdateFinalizerObjIdTables { + fn process_table(&mut self) { + (crate::upcalls().update_finalizer_table)(); + } +} +impl GCWork<Ruby> for UpdateFinalizerObjIdTables { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobalTableProcessingWork::do_work(self, worker, mmtk); + } +} + +struct UpdateGlobalTables { + idx: i32, +} +impl GlobalTableProcessingWork for UpdateGlobalTables { + fn process_table(&mut self) { + (crate::upcalls().update_global_tables)( + self.idx, + crate::mmtk().get_plan().current_gc_may_move_object(), + ) + } +} +impl GCWork<Ruby> for UpdateGlobalTables { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobalTableProcessingWork::do_work(self, worker, mmtk); + } +} + +struct UpdateWbUnprotectedObjectsList; + +impl GCWork<Ruby> for UpdateWbUnprotectedObjectsList { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut objects = crate::binding().wb_unprotected_objects.try_lock().expect( + "Someone is holding the lock of wb_unprotected_objects during weak processing phase?", + ); + + let old_objects = std::mem::take(&mut *objects); + + debug!("Updating {} WB-unprotected objects", old_objects.len()); + + for object in old_objects { + if object.is_reachable() { + // Forward and add back to the candidate list. + let new_object = object.forward(); + trace!("Forwarding WB-unprotected object: {object} -> {new_object}"); + objects.insert(new_object); + } else { + trace!("Removing WB-unprotected object from list: {object}"); + } + } + + debug!("Retained {} live WB-unprotected objects.", objects.len()); + } +} + +// Provide a shorthand `object.forward()`. +trait Forwardable { + fn forward(&self) -> Self; +} + +impl Forwardable for ObjectReference { + fn forward(&self) -> Self { + self.get_forwarded_object().unwrap_or(*self) + } +} diff --git a/gc/wbcheck/extconf.rb b/gc/wbcheck/extconf.rb new file mode 100644 index 0000000000..18b32d820d --- /dev/null +++ b/gc/wbcheck/extconf.rb @@ -0,0 +1,3 @@ +require_relative '../extconf_base' + +create_gc_makefile("wbcheck") diff --git a/gc/wbcheck/wbcheck.c b/gc/wbcheck/wbcheck.c new file mode 100644 index 0000000000..a7d4cd6ccf --- /dev/null +++ b/gc/wbcheck/wbcheck.c @@ -0,0 +1,1936 @@ +#include "internal.h" +#include "ruby/ruby.h" +#include "ruby/assert.h" +#include "ruby/atomic.h" +#include "ruby/debug.h" +#include "ruby/internal/core/rbasic.h" +#include "ruby/st.h" +#include "internal/object.h" +#include "internal/array.h" +#include "internal/class.h" + +#include "ruby/thread.h" +#include "gc/gc.h" +#include "gc/gc_impl.h" + +#include <stdbool.h> +#include <stdarg.h> + +// Debug output control +static bool wbcheck_debug_enabled = false; + +// Verification after write barrier control +static bool wbcheck_verify_after_wb_enabled = false; + +// Useless write barrier warning control +static bool wbcheck_warn_useless_wb_enabled = false; + +static void +wbcheck_debug(const char *format, ...) +{ + if (!wbcheck_debug_enabled) return; + + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); +} + +#define WBCHECK_DEBUG(...) do { \ + if (wbcheck_debug_enabled) { \ + wbcheck_debug(__VA_ARGS__); \ + } \ +} while (0) + +static void +wbcheck_debug_obj_info_dump(VALUE obj) +{ + if (!wbcheck_debug_enabled) return; + char buff[0x100]; + fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); +} + +// Forward declaration +static void lock_and_maybe_gc(void *objspace_ptr); +static void force_gc(void *objspace_ptr); + +// Configure wbcheck from environment variables +static void +wbcheck_configure_from_env(void) +{ + // Configure debug output based on environment variable + const char *debug_env = getenv("WBCHECK_DEBUG"); + if (debug_env && (strcmp(debug_env, "1") == 0 || strcmp(debug_env, "true") == 0)) { + wbcheck_debug_enabled = true; + } + + // Configure verification after write barrier based on environment variable + const char *verify_after_wb_env = getenv("WBCHECK_VERIFY_AFTER_WB"); + if (verify_after_wb_env && (strcmp(verify_after_wb_env, "1") == 0 || strcmp(verify_after_wb_env, "true") == 0)) { + wbcheck_verify_after_wb_enabled = true; + } + + // Configure useless write barrier warnings based on environment variable + const char *warn_useless_wb_env = getenv("WBCHECK_WARN_USELESS_WB"); + if (warn_useless_wb_env && (strcmp(warn_useless_wb_env, "1") == 0 || strcmp(warn_useless_wb_env, "true") == 0)) { + wbcheck_warn_useless_wb_enabled = true; + } +} + +// Define same heap sizes as the default GC +static size_t heap_sizes[] = { + 32, + 40, + 48, + 56, + 64, + 72, + 80, + 96, + 128, + 160, + 256, + 512, + 640, + 768, + 1024, + 0 +}; + +#define HEAP_COUNT ((int)(sizeof(heap_sizes) / sizeof(heap_sizes[0])) - 1) +#define MAX_HEAP_SIZE (heap_sizes[(HEAP_COUNT) - 1]) + +// Object states for verification tracking +typedef enum { + WBCHECK_STATE_CLEAR, // Just allocated or writebarrier_remember, needs reference capture + WBCHECK_STATE_MARKED, // Has valid snapshot, ready for normal operation + WBCHECK_STATE_DIRTY // Has seen writebarrier since last snapshot, queued for verification +} wbcheck_object_state_t; + +// Tri-color marking colors +typedef enum { + WBCHECK_COLOR_WHITE, // Unmarked - will be swept + WBCHECK_COLOR_GRAY, // Marked but children not processed + WBCHECK_COLOR_BLACK // Marked and children processed +} wbcheck_color_t; + +// GC phases +typedef enum { + WBCHECK_PHASE_MUTATOR, // Normal execution + WBCHECK_PHASE_SNAPSHOT, // Collecting references for verification + WBCHECK_PHASE_FULL_GC // Marking objects during full GC +} wbcheck_phase_t; + +// List of objects +typedef struct { + VALUE *items; + size_t count; + size_t capacity; +} wbcheck_object_list_t; + +// Helper functions for object list +static wbcheck_object_list_t * +wbcheck_object_list_init_with_capacity(size_t capacity) +{ + wbcheck_object_list_t *list = calloc(1, sizeof(wbcheck_object_list_t)); + if (!list) rb_bug("wbcheck: failed to allocate object list structure"); + + if (capacity < 4) capacity = 4; + list->items = malloc(capacity * sizeof(VALUE)); + if (!list->items) rb_bug("wbcheck: failed to allocate object list array"); + list->capacity = capacity; + list->count = 0; + return list; +} + +static wbcheck_object_list_t * +wbcheck_object_list_init(void) +{ + return wbcheck_object_list_init_with_capacity(4); +} + +static void +wbcheck_object_list_append(wbcheck_object_list_t *list, VALUE obj) +{ + if (list->count >= list->capacity) { + size_t new_capacity = list->capacity == 0 ? 4 : list->capacity * 2; + VALUE *new_items = realloc(list->items, new_capacity * sizeof(VALUE)); + if (!new_items) rb_bug("wbcheck: failed to reallocate object list array"); + list->items = new_items; + list->capacity = new_capacity; + } + list->items[list->count++] = obj; +} + +static void +wbcheck_object_list_free(wbcheck_object_list_t *list) +{ + if (!list) return; + if (list->items) { + free(list->items); + } + free(list); +} + +static void +wbcheck_object_list_debug_print(wbcheck_object_list_t *list) +{ + if (!wbcheck_debug_enabled) return; + for (size_t i = 0; i < list->count; i++) { + char buff[0x100]; + fprintf(stderr, "-> %s\n", rb_raw_obj_info(buff, sizeof(buff), list->items[i])); + } +} + +static bool +wbcheck_object_list_contains(wbcheck_object_list_t *list, VALUE obj) +{ + for (size_t i = 0; i < list->count; i++) { + if (list->items[i] == obj) { + return true; + } + } + return false; +} + +// Information tracked for each object +typedef struct { + size_t alloc_size; // Allocated size (static) + bool wb_protected; // Write barrier protection status (static) + VALUE finalizers; // Ruby Array of finalizers like [finalizer1, finalizer2, ...] + wbcheck_object_list_t *gc_mark_snapshot; // Snapshot of references from last GC mark + wbcheck_object_list_t *mark_maybe_snapshot; // Conservative refs reported via mark_maybe; needed for liveness, not verifiable + wbcheck_object_list_t *writebarrier_children; // References added via write barriers since last snapshot + wbcheck_object_state_t state; // Current state in verification lifecycle + wbcheck_color_t color; // Tri-color marking color +} rb_wbcheck_object_info_t; + +// Finalizer job types +struct wbcheck_final_job { + struct wbcheck_final_job *next; + enum { + WBCHECK_FINAL_JOB_DFREE, + WBCHECK_FINAL_JOB_FINALIZE, + } kind; + union { + struct { + void (*func)(void *); + void *data; + } dfree; + struct { + VALUE finalizer_array; + } finalize; + } as; +}; + +// wbcheck objspace structure to track all objects +typedef struct { + st_table *object_table; // Hash table to track all allocated objects (VALUE -> rb_wbcheck_object_info_t*) + wbcheck_object_list_t *objects_to_capture; // Objects that need initial reference capture + wbcheck_object_list_t *objects_to_verify; // Objects that need verification after write barriers + wbcheck_object_list_t *current_refs; // Current list for collecting references during marking + wbcheck_object_list_t *current_maybe_refs; // Current list for collecting mark_maybe references during marking + wbcheck_object_list_t *mark_queue; // Queue of gray objects for tri-color marking + wbcheck_object_list_t *weak_references; // Objects holding weak references, found during marking + wbcheck_phase_t phase; // Current GC phase + bool gc_enabled; // Whether GC is allowed to run + bool gc_stress; // GC stress mode (run GC on every allocation) + size_t gc_threshold; // Trigger GC when object count reaches this + size_t missed_write_barrier_parents; // Number of parent objects with missed write barriers + size_t missed_write_barrier_children; // Total number of missed write barriers detected + size_t simulated_gc_count; // Simulated GC count incremented on each GC.start + bool measure_total_time; // Whether to accumulate :time in stats + struct wbcheck_final_job *finalizer_jobs; // Linked list of finalizer jobs + rb_nativethread_lock_t finalizer_lock; // Protects finalizer_jobs list + rb_postponed_job_handle_t finalizer_postponed_job; // Postponed job handle for finalizers +} rb_wbcheck_objspace_t; + +// Global objspace pointer for accessing from obj_slot_size function +static rb_wbcheck_objspace_t *wbcheck_global_objspace = NULL; + +// Forward declarations +static void wbcheck_foreach_object(rb_wbcheck_objspace_t *objspace, int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data), void *data); +static int wbcheck_verify_all_references_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data); +static int wbcheck_update_all_snapshots_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data); +static void wbcheck_run_finalizers_for_object(VALUE obj, rb_wbcheck_object_info_t *info); +static void gc_run_finalizers(void *data); +static void make_final_job(rb_wbcheck_objspace_t *objspace, VALUE obj, VALUE finalizer_array); + +// Helper functions for object tracking +static rb_wbcheck_object_info_t * +wbcheck_get_object_info(VALUE obj) +{ + // Objspace must be initialized by this point + GC_ASSERT(wbcheck_global_objspace); + + st_data_t value; + if (st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value)) { + return (rb_wbcheck_object_info_t *)value; + } + + fprintf(stderr, "wbcheck: object not found in tracking table\n"); + char buff[0x100]; + fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); + + // Force ASAN crash? + ((volatile VALUE *)obj)[0]; + + // Object not found in tracking table - this should never happen + rb_bug("wbcheck: object not found in tracking table"); +} + +static void +wbcheck_report_error(void *objspace_ptr, VALUE parent_obj, wbcheck_object_list_t *current_refs, wbcheck_object_list_t *gc_mark_snapshot, wbcheck_object_list_t *writebarrier_children, wbcheck_object_list_t *missed_refs) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + rb_wbcheck_object_info_t *parent_info = wbcheck_get_object_info(parent_obj); + + size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0; + size_t wb_count = writebarrier_children ? writebarrier_children->count : 0; + + fprintf(stderr, "WBCHECK ERROR: Missed write barrier detected!\n"); + fprintf(stderr, " Parent object: %p (wb_protected: %s)\n", + (void *)parent_obj, parent_info->wb_protected ? "true" : "false"); + char buff[0x100]; + fprintf(stderr, " %s\n", rb_raw_obj_info(buff, sizeof(buff), parent_obj)); + fprintf(stderr, " Reference counts - snapshot: %zu, writebarrier: %zu, current: %zu, missed: %zu\n", + snapshot_count, wb_count, current_refs->count, missed_refs->count); + + for (size_t i = 0; i < missed_refs->count; i++) { + VALUE missed_ref = missed_refs->items[i]; + char buff[0x100]; + fprintf(stderr, " Missing reference to: %p\n %s\n", (void *)missed_ref, rb_raw_obj_info(buff, sizeof(buff), missed_ref)); + } + + fprintf(stderr, "\n"); + objspace->missed_write_barrier_parents++; + objspace->missed_write_barrier_children += missed_refs->count; +} + +static void +wbcheck_compare_references(void *objspace_ptr, VALUE parent_obj, wbcheck_object_list_t *current_refs, wbcheck_object_list_t *gc_mark_snapshot, wbcheck_object_list_t *writebarrier_children) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + (void)objspace; + + size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0; + size_t wb_count = writebarrier_children ? writebarrier_children->count : 0; + + WBCHECK_DEBUG("wbcheck: comparing references for object %p\n", (void *)parent_obj); + WBCHECK_DEBUG("wbcheck: current refs: %zu, snapshot refs: %zu, wb refs: %zu\n", + current_refs->count, snapshot_count, wb_count); + + // Collect missed references (lazily allocated) + wbcheck_object_list_t *missed_refs = NULL; + + // Use circular comparison for better performance when lists are mostly similar + size_t snapshot_idx = 0; + + // Check each object in current_refs to see if it's in either stored list + for (size_t i = 0; i < current_refs->count; i++) { + VALUE current_ref = current_refs->items[i]; + + // Usually the lists are nearly identical. We take advantage of this by + // attempting to loop over both lists in sequence. When the next element + // of the snapshot doesn't match the next element of our current_refs, + // we'll loop around the list to try to find it and continue from that + // match, so any runs of identical items can be matched efficiently. + // + // Pathologically this is O(N**2), but is O(N * num_changes) + bool found_in_snapshot = false; + if (gc_mark_snapshot && snapshot_count > 0) { + size_t start_idx = snapshot_idx; + do { + if (gc_mark_snapshot->items[snapshot_idx] == current_ref) { + found_in_snapshot = true; + snapshot_idx++; + if (snapshot_idx >= snapshot_count) snapshot_idx = 0; + break; + } + snapshot_idx++; + if (snapshot_idx >= snapshot_count) snapshot_idx = 0; + } while (snapshot_idx != start_idx); + } + + if (found_in_snapshot) { + continue; + } + + // Built-in immortal classes can be assigned via RBASIC_SET_CLASS_RAW, + // which bypasses the write barrier. They're pinned as VM roots and + // can never be collected, so a missing WB to them is harmless. + if (RB_TYPE_P(current_ref, T_CLASS) && FL_TEST_RAW(current_ref, RCLASS_IS_ROOT)) { + continue; + } + + // Self reference... Weird but okay I guess + if (current_ref == parent_obj) { + continue; + } + + + // Check if reference exists in writebarrier_children + if (writebarrier_children && wbcheck_object_list_contains(writebarrier_children, current_ref)) { + continue; + } + + // If we get here, the reference wasn't found in either list + // Lazily allocate missed_refs list on first miss + if (!missed_refs) { + missed_refs = wbcheck_object_list_init(); + } + wbcheck_object_list_append(missed_refs, current_ref); + } + + // Report any errors found + if (missed_refs) { + wbcheck_report_error(objspace_ptr, parent_obj, current_refs, gc_mark_snapshot, writebarrier_children, missed_refs); + wbcheck_object_list_free(missed_refs); + } +} + +static void +wbcheck_register_object(void *objspace_ptr, VALUE obj, size_t alloc_size, bool wb_protected) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + // Allocate and initialize object info structure + rb_wbcheck_object_info_t *info = calloc(1, sizeof(rb_wbcheck_object_info_t)); + if (!info) rb_bug("wbcheck_register_object: failed to allocate object info"); + + info->alloc_size = alloc_size; + info->wb_protected = wb_protected; + info->finalizers = 0; /* No finalizers initially */ + info->gc_mark_snapshot = NULL; /* No snapshot initially */ + info->mark_maybe_snapshot = NULL; /* No mark_maybe snapshot initially */ + info->writebarrier_children = NULL; /* No write barrier children initially */ + info->state = WBCHECK_STATE_CLEAR; /* Start in clear state */ + info->color = WBCHECK_COLOR_BLACK; /* Start as black to survive current GC */ + + // Store object info in hash table (VALUE -> rb_wbcheck_object_info_t*) + st_insert(objspace->object_table, (st_data_t)obj, (st_data_t)info); +} + +static void +wbcheck_unregister_object(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + rb_wbcheck_object_info_t *info; + + if (st_delete(objspace->object_table, (st_data_t *)&obj, (st_data_t *)&info)) { + // Free object lists if they were allocated + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->mark_maybe_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + free(info); + } else { + rb_bug("wbcheck_unregister_object: object not found in table"); + } +} + +// Bootup +void * +rb_gc_impl_objspace_alloc(void) +{ + wbcheck_configure_from_env(); + + rb_wbcheck_objspace_t *objspace = calloc(1, sizeof(rb_wbcheck_objspace_t)); + if (!objspace) rb_bug("wbcheck: failed to allocate objspace"); + + objspace->object_table = st_init_numtable(); + if (!objspace->object_table) { + free(objspace); + rb_bug("wbcheck: failed to create object table"); + } + + objspace->objects_to_capture = wbcheck_object_list_init(); // Initialize empty list + objspace->objects_to_verify = wbcheck_object_list_init(); // Initialize empty list + objspace->current_refs = NULL; // No current refs initially + objspace->current_maybe_refs = NULL; // No current maybe refs initially + objspace->mark_queue = wbcheck_object_list_init(); // Initialize mark queue + objspace->weak_references = wbcheck_object_list_init(); // Initialize weak references array + objspace->phase = WBCHECK_PHASE_MUTATOR; // Start in mutator phase + objspace->gc_enabled = true; // GC enabled by default (like default GC) + objspace->gc_stress = false; // GC stress disabled by default + objspace->gc_threshold = 1000; // Start with 1000 objects, will adjust after first GC + objspace->missed_write_barrier_parents = 0; // No errors found yet + objspace->missed_write_barrier_children = 0; // No errors found yet + objspace->simulated_gc_count = 0; // Start with GC count of 0 + objspace->measure_total_time = true; // On by default + + return objspace; +} + +void +rb_gc_impl_objspace_init(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + // Object table is already initialized in objspace_alloc + // Set up global objspace pointer for obj_slot_size function + wbcheck_global_objspace = objspace; + + // Initialize postponed job for finalizers + rb_native_mutex_initialize(&objspace->finalizer_lock); + objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace); +} + +void * +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) +{ + // Stub implementation + return NULL; +} + +void +rb_gc_impl_set_params(void *objspace_ptr) +{ + // Stub implementation +} + +static VALUE +gc_verify_internal_consistency(VALUE self) +{ + return Qnil; +} + +void +rb_gc_impl_init(void) +{ + VALUE gc_constants = rb_hash_new(); + //rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MAX_HEAP_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(3)); + OBJ_FREEZE(gc_constants); + rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants); + + // no-ops for compatibility + rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0); + + rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); + // Stub implementation +} + +size_t * +rb_gc_impl_heap_sizes(void *objspace_ptr) +{ + return heap_sizes; +} + +// Shutdown +void +rb_gc_impl_shutdown_free_objects(void *objspace_ptr) +{ + // Stub implementation +} + +void +rb_gc_impl_objspace_free(void *objspace_ptr) +{ + // This should free everything, but we'll just let it leak +} + +void +rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache) +{ + // Stub implementation +} + +// GC +void +rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + if (objspace) { + objspace->simulated_gc_count++; + } + + if (!ruby_native_thread_p()) return; + + unsigned int lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + force_gc(objspace_ptr); + RB_GC_VM_UNLOCK(lev); +} + +bool +rb_gc_impl_during_gc_p(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->phase != WBCHECK_PHASE_MUTATOR; +} + +static void +wbcheck_prepare_heap_i(VALUE obj, void *data) +{ + rb_gc_prepare_heap_process_object(obj); +} + +void +rb_gc_impl_prepare_heap(void *objspace_ptr) +{ + rb_gc_impl_each_object(objspace_ptr, wbcheck_prepare_heap_i, NULL); +} + +void +rb_gc_impl_gc_enable(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->gc_enabled = true; +} + +void +rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->gc_enabled = false; +} + +bool +rb_gc_impl_gc_enabled_p(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->gc_enabled; +} + +void +rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->gc_stress = RTEST(flag); +} + +VALUE +rb_gc_impl_stress_get(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->gc_stress ? Qtrue : Qfalse; +} + +VALUE +rb_gc_impl_config_get(void *objspace_ptr) +{ + return rb_hash_new(); +} + +void +rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) +{ +} + +static wbcheck_object_list_t * +wbcheck_collect_references_from_object(VALUE obj, rb_wbcheck_object_info_t *info) +{ + rb_wbcheck_objspace_t *objspace = wbcheck_global_objspace; + + // Use combination of writebarrier children and last snapshot as capacity hint + size_t snapshot_count = (info->gc_mark_snapshot) ? info->gc_mark_snapshot->count : 0; + size_t wb_children_count = (info->writebarrier_children) ? info->writebarrier_children->count : 0; + size_t capacity_hint = snapshot_count + wb_children_count; + wbcheck_object_list_t *new_list = wbcheck_object_list_init_with_capacity(capacity_hint); + + // Set up objspace state for marking. current_maybe_refs is allocated lazily + // by rb_gc_impl_mark_maybe, since most objects have no conservative refs. + objspace->current_refs = new_list; + objspace->current_maybe_refs = NULL; + objspace->phase = WBCHECK_PHASE_SNAPSHOT; + + // Use the marking infrastructure to collect references + rb_gc_mark_children(objspace, obj); + + // Clean up objspace state + objspace->phase = WBCHECK_PHASE_MUTATOR; + objspace->current_refs = NULL; + + // Update the mark_maybe snapshot in place. These references don't participate + // in verification, but we need to keep them so full GC can mark them gray. + wbcheck_object_list_free(info->mark_maybe_snapshot); + info->mark_maybe_snapshot = objspace->current_maybe_refs; + objspace->current_maybe_refs = NULL; + + if (wbcheck_debug_enabled) { + WBCHECK_DEBUG("wbcheck: collected %zu references from %p\n", new_list->count, (void *)obj); + char buff[0x100]; + fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); + wbcheck_object_list_debug_print(new_list); + } + + return new_list; +} + +static void +wbcheck_collect_initial_references(void *objspace_ptr, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: collecting initial references from %p:\n", obj); + wbcheck_debug_obj_info_dump(obj); + + // Get the object info and set the initial GC mark snapshot + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + wbcheck_object_list_t *new_list = wbcheck_collect_references_from_object(obj, info); + RUBY_ASSERT(!info->gc_mark_snapshot); + RUBY_ASSERT(info->state == WBCHECK_STATE_CLEAR); + info->gc_mark_snapshot = new_list; // Set the initial snapshot + info->state = WBCHECK_STATE_MARKED; // Transition to marked state +} + +static void +wbcheck_verify_object_references(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + // Ignore objects which are not write barrier protected + if (!info->wb_protected) { + return; + } + + // We hadn't captured initial references + if (info->state == WBCHECK_STATE_CLEAR) { + RUBY_ASSERT(!info->gc_mark_snapshot); + return; + } + + WBCHECK_DEBUG("wbcheck: verifying references for object:\n"); + wbcheck_debug_obj_info_dump(obj); + + // Get the current references from the object + wbcheck_object_list_t *current_refs = wbcheck_collect_references_from_object(obj, info); + + // Check for useless write barriers before clearing them + if (wbcheck_warn_useless_wb_enabled && info->writebarrier_children) { + for (size_t i = 0; i < info->writebarrier_children->count; i++) { + VALUE wb_ref = info->writebarrier_children->items[i]; + if (!wbcheck_object_list_contains(current_refs, wb_ref)) { + fprintf(stderr, "WBCHECK WARNING: Potentially useless write barrier detected for object %p\n", (void *)obj); + fprintf(stderr, " Write barrier was recorded for reference to %p, but object no longer references it\n", (void *)wb_ref); + char buff[0x100]; + fprintf(stderr, " Parent: %s\n", rb_raw_obj_info(buff, sizeof(buff), obj)); + fprintf(stderr, " Stale reference: %s\n", rb_raw_obj_info(buff, sizeof(buff), wb_ref)); + } + } + } + + // Compare current_refs against both stored lists to detect missed write barriers + wbcheck_compare_references(objspace_ptr, obj, current_refs, info->gc_mark_snapshot, info->writebarrier_children); + + // Update the snapshot with current references and clear write barrier children + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + info->gc_mark_snapshot = current_refs; + info->writebarrier_children = NULL; + info->state = WBCHECK_STATE_MARKED; // Back to marked state after verification +} + +// Mark object as gray (add to mark queue) +static void +wbcheck_mark_gray(rb_wbcheck_objspace_t *objspace, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + st_data_t value; + if (!st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_bug("wbcheck: asked to mark object %p not in our object table", (void *)obj); + } + + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + if (info->color != WBCHECK_COLOR_WHITE) { + return; // Already marked + } + + info->color = WBCHECK_COLOR_GRAY; + wbcheck_object_list_append(objspace->mark_queue, obj); + + if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) { + wbcheck_object_list_append(objspace->weak_references, obj); + } + + WBCHECK_DEBUG("wbcheck: marked gray: %p\n", (void *)obj); +} + +// Reset all objects to white +static int +st_foreach_reset_white(st_data_t key, st_data_t val, st_data_t arg) +{ + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + info->color = WBCHECK_COLOR_WHITE; + return ST_CONTINUE; +} + +// Mark all finalizer arrays to keep them alive during GC +static int +st_foreach_mark_finalizers(st_data_t key, st_data_t val, st_data_t arg) +{ + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)arg; + + if (info->finalizers) { + wbcheck_mark_gray(objspace, info->finalizers); + } + + return ST_CONTINUE; +} + +// Full mark phase using tri-color marking with snapshots +static void +wbcheck_mark_phase(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: starting GC mark phase\n"); + + objspace->phase = WBCHECK_PHASE_FULL_GC; + + // Clear mark queue and reset all objects to white + objspace->mark_queue->count = 0; + st_foreach(objspace->object_table, st_foreach_reset_white, 0); + + // Mark all finalizer arrays first to keep them alive + st_foreach(objspace->object_table, st_foreach_mark_finalizers, (st_data_t)objspace); + + // Mark finalizer arrays in pending jobs to keep them alive. + // No lock needed: all other threads are stopped during GC. + struct wbcheck_final_job *job = objspace->finalizer_jobs; + while (job != NULL) { + switch (job->kind) { + case WBCHECK_FINAL_JOB_DFREE: + break; + case WBCHECK_FINAL_JOB_FINALIZE: + wbcheck_mark_gray(objspace, job->as.finalize.finalizer_array); + break; + default: + rb_bug("wbcheck_mark_phase: unknown final job type %d", job->kind); + } + job = job->next; + } + + // Mark roots gray + rb_gc_save_machine_context(); + rb_gc_mark_roots(objspace, NULL); + + // Process gray queue until empty + while (objspace->mark_queue->count > 0) { + // Get last object from queue (LIFO) + VALUE obj = objspace->mark_queue->items[--objspace->mark_queue->count]; + + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + if (info->color == WBCHECK_COLOR_GRAY) { + // Mark all children from snapshot gray + if (info->gc_mark_snapshot) { + for (size_t i = 0; i < info->gc_mark_snapshot->count; i++) { + wbcheck_mark_gray(objspace, info->gc_mark_snapshot->items[i]); + } + } + + // Conservatively-scanned children must also be kept alive + if (info->mark_maybe_snapshot) { + for (size_t i = 0; i < info->mark_maybe_snapshot->count; i++) { + wbcheck_mark_gray(objspace, info->mark_maybe_snapshot->items[i]); + } + } + + // Mark this object black + info->color = WBCHECK_COLOR_BLACK; + WBCHECK_DEBUG("wbcheck: marked black: %p\n", (void *)obj); + } + } + } + + objspace->phase = WBCHECK_PHASE_MUTATOR; + + WBCHECK_DEBUG("wbcheck: tri-color mark phase complete\n"); +} + +// Sweep phase callback - free white objects +static int +wbcheck_sweep_callback(st_data_t key, st_data_t val, st_data_t arg, int error) +{ + VALUE obj = (VALUE)key; + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)arg; + + if (info->color == WBCHECK_COLOR_WHITE) { + WBCHECK_DEBUG("wbcheck: sweeping unmarked object %p\n", (void *)obj); + + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); + + // Clear weak references first + rb_gc_obj_free_vm_weak_references(obj); + + // Queue finalizers for postponed job if they exist + if (info->finalizers) { + make_final_job(objspace, obj, info->finalizers); + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } + + // Call rb_gc_obj_free which handles finalizers/zombies + if (rb_gc_obj_free(objspace, obj)) { + // Object was actually freed, clean up our tracking + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->mark_maybe_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + free(info); + + // Free the actual object memory + free((void *)obj); + + return ST_DELETE; // Remove from hash table + } else { + // Object became a zombie - it will be freed by postponed job + // Remove from tracking since we can't safely access it anymore + wbcheck_object_list_free(info->gc_mark_snapshot); + wbcheck_object_list_free(info->mark_maybe_snapshot); + wbcheck_object_list_free(info->writebarrier_children); + free(info); + + // Free the actual object memory + free((void *)obj); + + return ST_DELETE; // Remove from hash table + } + } + + return ST_CONTINUE; // Keep marked objects +} + +static void +wbcheck_sweep_phase(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: starting sweep phase\n"); + + size_t objects_before = st_table_size(objspace->object_table); + + // Sweep unmarked objects + st_foreach_check(objspace->object_table, wbcheck_sweep_callback, (st_data_t)objspace, 0); + + size_t objects_after = st_table_size(objspace->object_table); + size_t freed_objects = objects_before - objects_after; + + // Update GC threshold: 2x the live set after GC + objspace->gc_threshold = objects_after * 2; + + WBCHECK_DEBUG("wbcheck: sweep phase complete - freed %zu objects (%zu -> %zu), new threshold: %zu\n", + freed_objects, objects_before, objects_after, objspace->gc_threshold); +} + +// Process weak references after marking - call rb_gc_handle_weak_references +// on each object that was flagged with RUBY_FL_WEAK_REFERENCE and collected +// during the mark phase. +static void +wbcheck_process_weak_references(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: processing %zu weak reference objects\n", objspace->weak_references->count); + + for (size_t i = 0; i < objspace->weak_references->count; i++) { + VALUE obj = objspace->weak_references->items[i]; + rb_gc_handle_weak_references(obj); + } + + objspace->weak_references->count = 0; +} + +// Full GC: verify all objects then mark from roots +static void +wbcheck_full_gc(rb_wbcheck_objspace_t *objspace) +{ + WBCHECK_DEBUG("wbcheck: starting full GC\n"); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_ENTER); + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START); + + // First, update snapshots for all objects (verify wb_protected ones) + WBCHECK_DEBUG("wbcheck: updating snapshots for all objects\n"); + wbcheck_foreach_object(objspace, wbcheck_update_all_snapshots_callback, objspace); + + // Now start tri-color marking + wbcheck_mark_phase(objspace); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_MARK); + + // Process weak references after marking, before sweeping + wbcheck_process_weak_references(objspace); + + // Sweep unmarked objects + wbcheck_sweep_phase(objspace); + + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP); + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_EXIT); + + WBCHECK_DEBUG("wbcheck: full GC complete\n"); +} + +static void +gc_step(void *objspace_ptr, bool force) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + // Not initialized yet + if (!objspace) return; + + if (!objspace->gc_enabled && !force) return; + + // Process all objects that need verification after write barriers (if enabled) + if (wbcheck_verify_after_wb_enabled) { + for (size_t i = 0; i < objspace->objects_to_verify->count; i++) { + VALUE obj = objspace->objects_to_verify->items[i]; + wbcheck_verify_object_references(objspace_ptr, obj); + } + + // Clear the list after processing + objspace->objects_to_verify->count = 0; + + // If any new errors were detected during verification, exit immediately + if (objspace->missed_write_barrier_parents > 0) { + rb_bug("wbcheck: missed write barrier detected during immediate verification (WBCHECK_VERIFY_AFTER_WB=1)"); + } + } + + // Process all objects that need initial reference capture + for (size_t i = 0; i < objspace->objects_to_capture->count; i++) { + VALUE obj = objspace->objects_to_capture->items[i]; + wbcheck_collect_initial_references(objspace_ptr, obj); + } + + // Clear the list after processing + objspace->objects_to_capture->count = 0; + + // Run full GC if forced, if we exceed the threshold, or if gc_stress is enabled + if (ruby_native_thread_p() && + (force || + (objspace->gc_enabled && + (objspace->gc_stress || st_table_size(objspace->object_table) >= objspace->gc_threshold)))) { + wbcheck_full_gc(objspace); + } + +} + +static void +maybe_gc(void *objspace_ptr) +{ + gc_step(objspace_ptr, false); +} + +static void +force_gc(void *objspace_ptr) +{ + gc_step(objspace_ptr, true); +} + +int ruby_thread_has_gvl_p(void); + +static void * +lock_and_maybe_gc_gvl(void *objspace_ptr) +{ + unsigned int lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + maybe_gc(objspace_ptr); + + RB_GC_VM_UNLOCK(lev); + return NULL; +} + +static void +lock_and_maybe_gc(void *objspace_ptr) +{ + if (!ruby_native_thread_p()) return; + + if (!ruby_thread_has_gvl_p()) { + rb_thread_call_with_gvl(lock_and_maybe_gc_gvl, objspace_ptr); + } + else { + lock_and_maybe_gc_gvl(objspace_ptr); + } +} + +VALUE +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) +{ + unsigned int lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + // Check if we should trigger GC before allocating + maybe_gc(objspace_ptr); + + // Ensure minimum allocation size of BASE_SLOT_SIZE + alloc_size = heap_sizes[rb_gc_impl_heap_id_for_size(objspace_ptr, alloc_size)]; + + // Allocate memory for the object + VALUE *mem = malloc(alloc_size); + if (!mem) rb_bug("FIXME: malloc failed"); + + // Initialize the object + VALUE obj = (VALUE)mem; + RBASIC(obj)->flags = flags; + *((VALUE *)&RBASIC(obj)->klass) = klass; + + // Register the new object in our tracking table + wbcheck_register_object(objspace_ptr, obj, alloc_size, wb_protected); + + // Add this object to the list of objects that need initial reference capture + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + wbcheck_object_list_append(objspace->objects_to_capture, obj); + + RB_GC_VM_UNLOCK(lev); + return obj; +} + +size_t +rb_gc_impl_obj_slot_size(VALUE obj) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + size_t result = info->alloc_size; + + RB_GC_VM_UNLOCK(lev); + return result; +} + +size_t +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) +{ + for (int i = 0; i < HEAP_COUNT; i++) { + if (size <= heap_sizes[i]) return i; + } + rb_bug("size too big"); +} + +bool +rb_gc_impl_size_allocatable_p(size_t size) +{ + // Only allow sizes up to the largest heap size + return size <= MAX_HEAP_SIZE; +} + +// Malloc +void * +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + if (gc_allowed) { + lock_and_maybe_gc(objspace_ptr); + } + return malloc(size); +} + +void * +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + if (gc_allowed) { + lock_and_maybe_gc(objspace_ptr); + } + return calloc(1, size); +} + +void * +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) +{ + if (gc_allowed) { + lock_and_maybe_gc(objspace_ptr); + } + return realloc(ptr, new_size); +} + +void +rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) +{ + free(ptr); +} + +void +rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) +{ + // For wbcheck, we don't track memory usage +} + +// Marking +static void +gc_mark(rb_wbcheck_objspace_t *objspace, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: gc_mark called\n"); + wbcheck_debug_obj_info_dump(obj); + + if (RB_SPECIAL_CONST_P(obj)) return; + + switch (objspace->phase) { + case WBCHECK_PHASE_SNAPSHOT: + // Collecting references during verification + GC_ASSERT(objspace->current_refs); + wbcheck_object_list_append(objspace->current_refs, obj); + break; + case WBCHECK_PHASE_FULL_GC: + // Marking during full GC + wbcheck_mark_gray(objspace, obj); + break; + case WBCHECK_PHASE_MUTATOR: + // Should not be called during mutator phase + rb_bug("wbcheck: gc_mark called during mutator phase"); + break; + } +} + +void +rb_gc_impl_mark(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + gc_mark(objspace, obj); +} + +void +rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + gc_mark(objspace, *ptr); +} + +void +rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + gc_mark(objspace, obj); +} + +void +rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)obj)) return; + + switch (objspace->phase) { + case WBCHECK_PHASE_SNAPSHOT: + // We don't know if this is actually a reference or just a value + // that looks like one, so we can't expect a write barrier for it. + // Keep it separate from the verifiable refs, but retain it so full + // GC can mark the target gray if it does turn out to be live. + if (!objspace->current_maybe_refs) { + objspace->current_maybe_refs = wbcheck_object_list_init(); + } + wbcheck_object_list_append(objspace->current_maybe_refs, obj); + break; + case WBCHECK_PHASE_FULL_GC: + wbcheck_mark_gray(objspace, obj); + break; + case WBCHECK_PHASE_MUTATOR: + rb_bug("wbcheck: rb_gc_impl_mark_maybe called during mutator phase"); + break; + } +} + +// Weak references +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) +{ + FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE); +} + +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + return info->color != WBCHECK_COLOR_WHITE; + } + + return false; +} + +// Compaction +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + /* no-op */ +} + +bool +rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) +{ + // Stub implementation + return false; +} + +VALUE +rb_gc_impl_location(void *objspace_ptr, VALUE value) +{ + // Stub implementation + return Qnil; +} + +// Write barriers +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + if (RB_SPECIAL_CONST_P(b)) return; + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + // Get the object info for the parent object (a) + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(a); + + // Only record the write barrier if we have a valid snapshot + if (info->state != WBCHECK_STATE_CLEAR) { + RUBY_ASSERT(info->gc_mark_snapshot); + + // Initialize writebarrier_children list if it doesn't exist + if (!info->writebarrier_children) { + info->writebarrier_children = wbcheck_object_list_init(); + } + + // Add the new reference to the write barrier children list + wbcheck_object_list_append(info->writebarrier_children, b); + + WBCHECK_DEBUG("wbcheck: write barrier recorded reference from %p to %p\n", (void *)a, (void *)b); + + // If verification after write barrier is enabled, queue the object for verification + if (wbcheck_verify_after_wb_enabled && info->state != WBCHECK_STATE_DIRTY) { + WBCHECK_DEBUG("wbcheck: queueing object for verification after write barrier\n"); + info->state = WBCHECK_STATE_DIRTY; // Mark as dirty + wbcheck_object_list_append(objspace->objects_to_verify, a); + } + } else { + WBCHECK_DEBUG("wbcheck: write barrier skipped (snapshot not initialized) from %p to %p\n", (void *)a, (void *)b); + } + + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: writebarrier_unprotect called on object %p\n", (void *)obj); + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + info->wb_protected = false; + + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + WBCHECK_DEBUG("wbcheck: writebarrier_remember called on object %p\n", (void *)obj); + + unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER(); + + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + // Clear existing references since they may be stale + if (info->state != WBCHECK_STATE_CLEAR) { + RUBY_ASSERT(info->gc_mark_snapshot); + wbcheck_object_list_free(info->gc_mark_snapshot); + info->gc_mark_snapshot = NULL; + + wbcheck_object_list_free(info->mark_maybe_snapshot); + info->mark_maybe_snapshot = NULL; + + // Only re-add to objects_to_capture if it had previous snapshot + // (new objects don't need to be re-added since they'll be captured at allocation) + wbcheck_object_list_append(objspace->objects_to_capture, obj); + + // Also clear write barrier children + if (info->writebarrier_children) { + wbcheck_object_list_free(info->writebarrier_children); + info->writebarrier_children = NULL; + } + + // Reset to clear state + info->state = WBCHECK_STATE_CLEAR; + } + RUBY_ASSERT(!info->gc_mark_snapshot); + RUBY_ASSERT(!info->mark_maybe_snapshot); + RUBY_ASSERT(!info->writebarrier_children); + + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +// Heap walking +struct wbcheck_foreach_data { + int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data); + void *data; +}; + +static int +wbcheck_foreach_object_i(st_data_t key, st_data_t val, st_data_t arg) +{ + VALUE obj = (VALUE)key; + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val; + struct wbcheck_foreach_data *foreach_data = (struct wbcheck_foreach_data *)arg; + + return foreach_data->callback(obj, info, foreach_data->data); +} + +static void +wbcheck_foreach_object(rb_wbcheck_objspace_t *objspace, int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data), void *data) +{ + struct wbcheck_foreach_data foreach_data = { + .callback = callback, + .data = data + }; + + st_foreach(objspace->object_table, wbcheck_foreach_object_i, (st_data_t)&foreach_data); +} + +// Helper to collect all objects into a snapshot list +static int +wbcheck_snapshot_collector(st_data_t key, st_data_t val, st_data_t arg) +{ + VALUE obj = (VALUE)key; + wbcheck_object_list_t *snapshot = (wbcheck_object_list_t *)arg; + wbcheck_object_list_append(snapshot, obj); + return ST_CONTINUE; +} + +// Take a snapshot of all objects for safe iteration +static wbcheck_object_list_t * +wbcheck_take_object_snapshot(rb_wbcheck_objspace_t *objspace) +{ + size_t object_count = st_table_size(objspace->object_table); + wbcheck_object_list_t *snapshot = wbcheck_object_list_init_with_capacity(object_count); + st_foreach(objspace->object_table, wbcheck_snapshot_collector, (st_data_t)snapshot); + return snapshot; +} + + +void +rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + wbcheck_object_list_t *snapshot = wbcheck_take_object_snapshot(objspace); + + for (size_t i = 0; i < snapshot->count; i++) { + VALUE obj = snapshot->items[i]; + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value; + int result = callback( + (void *)obj, + (void *)((char *)obj + info->alloc_size), + info->alloc_size, + data + ); + if (result != 0) break; + } + } + + wbcheck_object_list_free(snapshot); +} + +void +rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + wbcheck_object_list_t *snapshot = wbcheck_take_object_snapshot(objspace); + + for (size_t i = 0; i < snapshot->count; i++) { + VALUE obj = snapshot->items[i]; + st_data_t value; + if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) { + func(obj, data); + } + } + + wbcheck_object_list_free(snapshot); +} + +static void +finalizer_jobs_push(rb_wbcheck_objspace_t *objspace, struct wbcheck_final_job *job) +{ + rb_native_mutex_lock(&objspace->finalizer_lock); + job->next = objspace->finalizer_jobs; + objspace->finalizer_jobs = job; + rb_native_mutex_unlock(&objspace->finalizer_lock); +} + +static struct wbcheck_final_job * +finalizer_jobs_pop(rb_wbcheck_objspace_t *objspace) +{ + rb_native_mutex_lock(&objspace->finalizer_lock); + struct wbcheck_final_job *job = objspace->finalizer_jobs; + if (job) { + objspace->finalizer_jobs = job->next; + } + rb_native_mutex_unlock(&objspace->finalizer_lock); + return job; +} + +// Finalizers +void +rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) +{ + if (dfree == NULL) return; + + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + + struct wbcheck_final_job *job = malloc(sizeof(struct wbcheck_final_job)); + job->kind = WBCHECK_FINAL_JOB_DFREE; + job->as.dfree.func = dfree; + job->as.dfree.data = data; + + finalizer_jobs_push(objspace, job); + + if (!ruby_free_at_exit_p()) { + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } + + WBCHECK_DEBUG("wbcheck: made zombie for object %p with dfree function\n", (void *)obj); +} + +VALUE +rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + (void)objspace_ptr; + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + GC_ASSERT(!OBJ_FROZEN(obj)); + + RBASIC(obj)->flags |= FL_FINALIZE; + + VALUE table = info->finalizers; + VALUE result = block; + + if (!table) { + /* First finalizer for this object - store object ID as first element */ + table = rb_ary_new3(2, rb_obj_id(obj), block); + rb_obj_hide(table); + info->finalizers = table; + } else { + /* Check for duplicate finalizers (skip index 0 which is object ID) */ + long len = RARRAY_LEN(table); + long i; + + for (i = 1; i < len; i++) { + VALUE recv = RARRAY_AREF(table, i); + if (rb_equal(recv, block)) { + result = recv; /* Duplicate found, return existing */ + goto unlock_and_return; + } + } + + rb_ary_push(table, block); + } + +unlock_and_return: + RB_GC_VM_UNLOCK(lev); + return result; +} + +void +rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + (void)objspace_ptr; + rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj); + + GC_ASSERT(!OBJ_FROZEN(obj)); + + info->finalizers = 0; + FL_UNSET(obj, FL_FINALIZE); + + RB_GC_VM_UNLOCK(lev); +} + +void +rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) +{ + (void)objspace_ptr; + + if (!FL_TEST(obj, FL_FINALIZE)) return; + + unsigned int lev = RB_GC_VM_LOCK(); + + rb_wbcheck_object_info_t *src_info = wbcheck_get_object_info(obj); + rb_wbcheck_object_info_t *dest_info = wbcheck_get_object_info(dest); + + if (src_info->finalizers) { + VALUE table = rb_ary_dup(src_info->finalizers); + RARRAY_ASET(table, 0, rb_obj_id(dest)); + rb_obj_hide(table); + dest_info->finalizers = table; + FL_SET(dest, FL_FINALIZE); + } + + RB_GC_VM_UNLOCK(lev); +} + +static VALUE +wbcheck_get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i + 1); +} + +static void +make_final_job(rb_wbcheck_objspace_t *objspace, VALUE obj, VALUE finalizer_array) +{ + RUBY_ASSERT(RB_FL_TEST(obj, FL_FINALIZE)); + RUBY_ASSERT(RB_BUILTIN_TYPE(finalizer_array) == T_ARRAY); + + RB_FL_UNSET(obj, FL_FINALIZE); + + struct wbcheck_final_job *job = malloc(sizeof(struct wbcheck_final_job)); + job->kind = WBCHECK_FINAL_JOB_FINALIZE; + job->as.finalize.finalizer_array = finalizer_array; + + finalizer_jobs_push(objspace, job); +} + +static void +gc_run_finalizers(void *data) +{ + rb_wbcheck_objspace_t *objspace = data; + + rb_gc_set_pending_interrupt(); + + struct wbcheck_final_job *job; + while ((job = finalizer_jobs_pop(objspace)) != NULL) { + switch (job->kind) { + case WBCHECK_FINAL_JOB_DFREE: + job->as.dfree.func(job->as.dfree.data); + break; + case WBCHECK_FINAL_JOB_FINALIZE: { + VALUE finalizer_array = job->as.finalize.finalizer_array; + + rb_gc_run_obj_finalizer( + RARRAY_AREF(finalizer_array, 0), + RARRAY_LEN(finalizer_array) - 1, + wbcheck_get_final, + (void *)finalizer_array + ); + + RB_GC_GUARD(finalizer_array); + break; + } + } + + free(job); + } + + rb_gc_unset_pending_interrupt(); +} + +static void +wbcheck_run_finalizers_for_object(VALUE obj, rb_wbcheck_object_info_t *info) +{ + if (info->finalizers) { + VALUE table = info->finalizers; + long count = RARRAY_LEN(table) - 1; + rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), count, wbcheck_get_final, (void *)table); + FL_UNSET(obj, FL_FINALIZE); + } + info->finalizers = 0; +} + +static int +wbcheck_shutdown_call_finalizer_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + wbcheck_run_finalizers_for_object(obj, info); + return ST_CONTINUE; /* Keep iterating through all objects */ +} + +static int +wbcheck_verify_all_references_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + void *objspace_ptr = data; + wbcheck_verify_object_references(objspace_ptr, obj); + return ST_CONTINUE; +} + +static int +wbcheck_update_all_snapshots_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + void *objspace_ptr = data; + + // For wb_protected objects, do full verification if they have a snapshot + if (info->wb_protected && info->state != WBCHECK_STATE_CLEAR) { + wbcheck_verify_object_references(objspace_ptr, obj); + } else { + // For CLEAR objects (wb_protected or not) and non-wb_protected objects, just take a new snapshot + wbcheck_object_list_t *current_refs = wbcheck_collect_references_from_object(obj, info); + wbcheck_object_list_free(info->gc_mark_snapshot); + info->gc_mark_snapshot = current_refs; + info->state = WBCHECK_STATE_MARKED; + } + + return ST_CONTINUE; +} + +static int +wbcheck_shutdown_finalizer_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data) +{ + void *objspace_ptr = data; + + if (rb_gc_shutdown_call_finalizer_p(obj)) { + WBCHECK_DEBUG("wbcheck: finalizing object during shutdown: %p\n", (void *)obj); + rb_gc_obj_free_vm_weak_references(obj); + if (rb_gc_obj_free(objspace_ptr, obj)) { + RBASIC(obj)->flags = 0; + } + } + + return ST_CONTINUE; +} + + +void +rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = objspace_ptr; + + // Call all finalizers for all objects using our shared iteration helper + wbcheck_foreach_object(objspace, wbcheck_shutdown_call_finalizer_callback, NULL); + + // After all finalizers have been called, verify all object references + unsigned int verify_lev = RB_GC_VM_LOCK(); + WBCHECK_DEBUG("wbcheck: verifying references for all objects after finalizers\n"); + wbcheck_foreach_object(objspace, wbcheck_verify_all_references_callback, objspace_ptr); + WBCHECK_DEBUG("wbcheck: finished verifying all object references\n"); + RB_GC_VM_UNLOCK(verify_lev); + + // Print summary and exit with error code if violations were found + if (objspace->missed_write_barrier_parents > 0 || objspace->missed_write_barrier_children > 0) { + fprintf(stderr, "WBCHECK SUMMARY: Found %zu objects with missed write barriers (%zu total violations)\n", + objspace->missed_write_barrier_parents, objspace->missed_write_barrier_children); + + + exit(1); // Exit with error code to indicate violations were found + } else { + WBCHECK_DEBUG("wbcheck: no write barrier violations detected\n"); + } + + // Call rb_gc_obj_free on objects that need shutdown finalization (File, Data with dfree, etc.) + unsigned int lev = RB_GC_VM_LOCK(); + WBCHECK_DEBUG("wbcheck: calling rb_gc_obj_free on objects that need shutdown finalization\n"); + wbcheck_foreach_object(objspace, wbcheck_shutdown_finalizer_callback, objspace_ptr); + WBCHECK_DEBUG("wbcheck: finished calling rb_gc_obj_free\n"); + + // Run any pending finalizer jobs (dfree functions) + WBCHECK_DEBUG("wbcheck: running pending finalizer jobs\n"); + gc_run_finalizers(objspace); + WBCHECK_DEBUG("wbcheck: finished running finalizer jobs\n"); + RB_GC_VM_UNLOCK(lev); +} + +// Forking +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + // Stub implementation +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + // Stub implementation +} + +// Statistics +void +rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + objspace->measure_total_time = RTEST(flag); +} + +bool +rb_gc_impl_get_measure_total_time(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->measure_total_time; +} + +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + return objspace->measure_total_time ? objspace->simulated_gc_count : 0; +} + +size_t +rb_gc_impl_gc_count(void *objspace_ptr) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + if (objspace) { + return objspace->simulated_gc_count; + } + return 0; +} + +VALUE +rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key) +{ + // Stub implementation + return Qnil; +} + +VALUE +rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) +{ + rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr; + GC_ASSERT(objspace); + + VALUE hash = Qnil, key = Qnil; + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + hash = hash_or_sym; + } + else if (SYMBOL_P(hash_or_sym)) { + key = hash_or_sym; + } + else { + rb_bug("non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == ID2SYM(rb_intern(#name))) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, ID2SYM(rb_intern(#name)), SIZET2NUM(attr)); + + /* Pretend each GC takes 1ms; :time is reported in milliseconds. */ + SET(count, objspace->simulated_gc_count); + SET(time, objspace->measure_total_time ? objspace->simulated_gc_count : 0); + SET(tracked_objects, st_table_size(objspace->object_table)); +#undef SET + + if (!NIL_P(key)) { + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); + } + + rb_hash_aset(hash, ID2SYM(rb_intern("gc_implementation")), rb_str_new_cstr("wbcheck")); + + return hash; +} + +VALUE +rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) +{ + if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) { + int heap_idx = FIX2INT(heap_name); + if (heap_idx < 0 || heap_idx >= HEAP_COUNT) { + rb_raise(rb_eArgError, "size pool index out of range"); + } + + if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) { + return SIZET2NUM(heap_sizes[heap_idx]); + } + + return Qundef; + } + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + return hash_or_sym; + } + + return Qundef; +} + +const char * +rb_gc_impl_active_gc_name(void) +{ + // Stub implementation + return "wbcheck"; +} + +// Miscellaneous +#define WBCHECK_OBJECT_METADATA_ENTRY_COUNT 2 +static struct rb_gc_object_metadata_entry object_metadata_entries[WBCHECK_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) +{ + static ID ID_object_id, ID_shareable; + + if (!ID_object_id) { + ID_object_id = rb_intern("object_id"); + ID_shareable = rb_intern("shareable"); + } + + size_t n = 0; + +#define SET_ENTRY(na, v) do { \ + GC_ASSERT(n < WBCHECK_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue); +#undef SET_ENTRY + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; + + return object_metadata_entries; +} + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + GC_ASSERT(wbcheck_global_objspace); + + unsigned int lev = RB_GC_VM_LOCK(); + + // Check if this pointer exists in our object tracking table + st_data_t value; + bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)ptr, &value); + + RB_GC_VM_UNLOCK(lev); + return result; +} + +bool +rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj) +{ + unsigned int lev = RB_GC_VM_LOCK(); + + // Check if this pointer exists in our object tracking table + st_data_t value; + bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value); + + RB_GC_VM_UNLOCK(lev); + return !result; +} + +void +rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) +{ + // Stub implementation +} + +void +rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) +{ + rb_wbcheck_object_info_t *src_info = wbcheck_get_object_info(obj); + + if (!src_info->wb_protected) { + rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest); + } + rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj); +} + |